diff --git a/.github/workflows/release-extension.yml b/.github/workflows/release-extension.yml index 381a67f81..880542e97 100644 --- a/.github/workflows/release-extension.yml +++ b/.github/workflows/release-extension.yml @@ -1,19 +1,55 @@ name: Release Lambda Extension +# Versioning tracks DataDog's upstream releases. The upstream version lives in +# the layer NAME (Tero-Datadog-Extension-119), and the AWS layer version integer +# is the patch number. An AWS layer version cannot be set, only appended, so +# putting the upstream number in the name is what keeps it exact — in every +# region, from the first publish, without burning filler versions. +# +# upstream v119, first release -> Tero-Datadog-Extension-119:1 +# patch on top of upstream v119 -> Tero-Datadog-Extension-119:2 +# +# Tag "v119" to release, "v119.2" to release a patch and assert it lands on 2. on: push: tags: - "v[0-9]+" - workflow_dispatch: + - "v[0-9]+.[0-9]+" + # Called by upstream-release-watch.yml. A tag pushed with GITHUB_TOKEN does not + # trigger another workflow, so the watch job invokes this one directly instead + # of relying on a tag push. github.event_name is the caller's event, so a + # scheduled watch run does NOT get the -dev suffix — it is a real release. + workflow_call: inputs: version: - description: "Version number (e.g., 100). If not provided, auto-increments from latest." + description: "Upstream version to release, e.g. 119 or 119.2 (patch optional)" + required: true + type: string + regions: + description: "Comma-separated AWS regions. Empty publishes to DEFAULT_REGIONS." + required: false + default: "" + type: string + architectures: + description: "Architectures to build and publish" required: false + default: "amd64,arm64" + type: string + dry_run: + description: "Dry run - build but don't publish to AWS" + required: false + default: false + type: boolean + workflow_dispatch: + inputs: + version: + description: "Upstream version to release, e.g. 119 or 119.2 (patch optional)" + required: true type: string regions: - description: "Comma-separated AWS regions to publish to (e.g., 'us-east-1,us-west-2')" + description: "Comma-separated AWS regions. Leave empty to publish to every region in DEFAULT_REGIONS." required: false - default: "us-east-1" + default: "" type: string architectures: description: "Architectures to build and publish" @@ -34,37 +70,57 @@ permissions: id-token: write contents: read +env: + # Every US and EU region that is enabled by default. Opt-in regions are left + # out on purpose: publishing to one that the account has not enabled fails the + # job. Add eu-south-1, eu-south-2 or eu-central-2 here once they are enabled. + DEFAULT_REGIONS: "us-east-1,us-east-2,us-west-1,us-west-2,eu-west-1,eu-west-2,eu-west-3,eu-central-1,eu-north-1" + jobs: prepare: name: Prepare release runs-on: ubuntu-22.04 outputs: - version: ${{ steps.version.outputs.version }} + upstream_version: ${{ steps.version.outputs.upstream_version }} + patch: ${{ steps.version.outputs.patch }} regions_json: ${{ steps.vars.outputs.regions_json }} build_matrix: ${{ steps.matrix.outputs.build_matrix }} publish_matrix: ${{ steps.matrix.outputs.publish_matrix }} steps: - uses: actions/checkout@v4 + # Accepts "119" or "119.2", with or without a leading "v". The patch is + # optional: when absent the publish step takes whatever AWS assigns next, + # which is what a first release into a new region needs. - name: Determine version id: version + # Passed through env, not interpolated into the script, so the input + # cannot inject shell. + env: + INPUT_VERSION: ${{ inputs.version }} run: | - if [ -n "${{ inputs.version }}" ]; then - # Strip 'v' prefix if present and extract number - VERSION=$(echo "${{ inputs.version }}" | sed 's/^v//') - echo "version=$VERSION" >> $GITHUB_OUTPUT - elif [[ "${{ github.ref }}" =~ ^refs/tags/v([0-9]+)$ ]]; then - echo "version=${BASH_REMATCH[1]}" >> $GITHUB_OUTPUT - else - # Auto-increment will happen at publish time per-region - echo "version=auto" >> $GITHUB_OUTPUT + set -euo pipefail + + RAW="$INPUT_VERSION" + if [ -z "$RAW" ]; then + RAW="${GITHUB_REF#refs/tags/}" fi + if ! PARSED=$(./scripts/parse_release_version.sh "$RAW"); then + echo "::error::Could not parse version '$RAW'" + exit 1 + fi + read -r UPSTREAM PATCH <<< "$PARSED" + + echo "upstream_version=$UPSTREAM" >> "$GITHUB_OUTPUT" + echo "patch=$PATCH" >> "$GITHUB_OUTPUT" + echo "Upstream version: $UPSTREAM, patch: ${PATCH:-}" + - name: Set variables id: vars run: | # Convert regions to JSON array (compact, single line) - REGIONS="${{ inputs.regions || 'us-east-1' }}" + REGIONS="${{ inputs.regions || env.DEFAULT_REGIONS }}" REGIONS_JSON=$(echo "$REGIONS" | jq -c -R 'split(",") | map(gsub("^\\s+|\\s+$";""))') echo "regions_json=$REGIONS_JSON" >> $GITHUB_OUTPUT echo "Regions: $REGIONS_JSON" @@ -73,7 +129,7 @@ jobs: id: matrix run: | ARCHS="${{ inputs.architectures || 'amd64,arm64' }}" - REGIONS="${{ inputs.regions || 'us-east-1' }}" + REGIONS="${{ inputs.regions || env.DEFAULT_REGIONS }}" # Build matrix for build job with platform-specific runners BUILD_MATRIX='{"include":[' @@ -213,8 +269,9 @@ jobs: DEV_SUFFIX="" fi - # Layer name: Tero-Datadog-Extension[-ARM][-dev] - LAYER_NAME="Tero-Datadog-Extension${ARCH_SUFFIX}${DEV_SUFFIX}" + # Layer name: Tero-Datadog-Extension-[-ARM][-dev] + UPSTREAM="${{ needs.prepare.outputs.upstream_version }}" + LAYER_NAME="Tero-Datadog-Extension-${UPSTREAM}${ARCH_SUFFIX}${DEV_SUFFIX}" echo "layer_name=$LAYER_NAME" >> $GITHUB_OUTPUT echo "compatible_arch=$COMPATIBLE_ARCH" >> $GITHUB_OUTPUT @@ -225,90 +282,71 @@ jobs: name: datadog_extension-${{ matrix.arch }} path: .layers - - name: Determine version + # A layer version can only be appended, never chosen, so a requested patch + # is checked BEFORE publishing. Checking afterwards would leave a wrongly + # numbered layer version behind, and those cannot be renumbered. + - name: Check target patch version is next id: version run: | - VERSION="${{ needs.prepare.outputs.version }}" + set -euo pipefail LAYER_NAME="${{ steps.vars.outputs.layer_name }}" + PATCH="${{ needs.prepare.outputs.patch }}" - if [ "$VERSION" = "auto" ]; then - # Get latest version and increment - LATEST=$(aws lambda list-layer-versions \ - --layer-name "$LAYER_NAME" \ - --query 'LayerVersions[0].Version' \ - --output text 2>/dev/null | head -1 || echo "0") - # Handle empty/None result - if [ -z "$LATEST" ] || [ "$LATEST" = "None" ]; then - LATEST=0 - fi - VERSION=$((LATEST + 1)) - echo "Auto-incremented version to $VERSION (latest was $LATEST)" + LATEST=$(aws lambda list-layer-versions \ + --layer-name "$LAYER_NAME" \ + --query 'LayerVersions[0].Version' \ + --output text 2>/dev/null | head -1 || echo "0") + if [ -z "$LATEST" ] || [ "$LATEST" = "None" ]; then + LATEST=0 fi + NEXT=$((LATEST + 1)) - echo "version=$VERSION" >> $GITHUB_OUTPUT + if [ -n "$PATCH" ] && [ "$PATCH" != "$NEXT" ]; then + echo "::error::$LAYER_NAME in ${{ matrix.region }} would publish patch $NEXT, but $PATCH was requested." + echo "::error::Latest published patch is $LATEST. Re-tag as v${{ needs.prepare.outputs.upstream_version }}.$NEXT." + exit 1 + fi + + echo "version=$NEXT" >> "$GITHUB_OUTPUT" + echo "Publishing $LAYER_NAME patch $NEXT to ${{ matrix.region }}" - name: Publish Lambda layer id: publish run: | + set -euo pipefail aws sts get-caller-identity LAYER_NAME="${{ steps.vars.outputs.layer_name }}" - VERSION="${{ steps.version.outputs.version }}" LAYER_FILE=".layers/datadog_extension-${{ matrix.arch }}.zip" COMPATIBLE_ARCH="${{ steps.vars.outputs.compatible_arch }}" + UPSTREAM="${{ needs.prepare.outputs.upstream_version }}" - echo "Publishing $LAYER_NAME version $VERSION to ${{ matrix.region }}..." - - # Check if version already exists - LATEST=$(aws lambda list-layer-versions \ + PUBLISHED_VERSION=$(aws lambda publish-layer-version \ --layer-name "$LAYER_NAME" \ - --query 'LayerVersions[0].Version' \ - --output text 2>/dev/null | head -1 || echo "0") - # Handle empty/None result - if [ -z "$LATEST" ] || [ "$LATEST" = "None" ]; then - LATEST=0 - fi - - if [ "$LATEST" -ge "$VERSION" ]; then - echo "::warning::Layer $LAYER_NAME version $VERSION already exists (latest: $LATEST), skipping" - echo "skipped=true" >> $GITHUB_OUTPUT - exit 0 - fi - - # Publish missing versions up to target - while [ "$LATEST" -lt "$VERSION" ]; do - PUBLISHED_VERSION=$(aws lambda publish-layer-version \ - --layer-name "$LAYER_NAME" \ - --description "Tero Datadog Lambda Extension" \ - --compatible-architectures "$COMPATIBLE_ARCH" \ - --zip-file "fileb://${LAYER_FILE}" \ - --query 'Version' \ - --output text) - - echo "Published version $PUBLISHED_VERSION" - - # Make the layer version public - aws lambda add-layer-version-permission \ - --layer-name "$LAYER_NAME" \ - --version-number "$PUBLISHED_VERSION" \ - --statement-id public-access \ - --action lambda:GetLayerVersion \ - --principal "*" - echo "Made version $PUBLISHED_VERSION public" - - LATEST=$PUBLISHED_VERSION - done + --description "Tero Datadog Lambda Extension (DataDog upstream v${UPSTREAM})" \ + --compatible-architectures "$COMPATIBLE_ARCH" \ + --zip-file "fileb://${LAYER_FILE}" \ + --query 'Version' \ + --output text) + echo "Published version $PUBLISHED_VERSION" + + # Make the layer version public + aws lambda add-layer-version-permission \ + --layer-name "$LAYER_NAME" \ + --version-number "$PUBLISHED_VERSION" \ + --statement-id public-access \ + --action lambda:GetLayerVersion \ + --principal "*" + echo "Made version $PUBLISHED_VERSION public" - echo "skipped=false" >> $GITHUB_OUTPUT - echo "published_version=$LATEST" >> $GITHUB_OUTPUT + echo "published_version=$PUBLISHED_VERSION" >> "$GITHUB_OUTPUT" # Output the layer ARN for reference ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - LAYER_ARN="arn:aws:lambda:${{ matrix.region }}:${ACCOUNT_ID}:layer:${LAYER_NAME}:${LATEST}" + LAYER_ARN="arn:aws:lambda:${{ matrix.region }}:${ACCOUNT_ID}:layer:${LAYER_NAME}:${PUBLISHED_VERSION}" echo "Layer ARN: $LAYER_ARN" - echo "layer_arn=$LAYER_ARN" >> $GITHUB_OUTPUT + echo "layer_arn=$LAYER_ARN" >> "$GITHUB_OUTPUT" - name: Summary - if: steps.publish.outputs.skipped != 'true' run: | echo "### Published Layer" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/upstream-release-watch.yml b/.github/workflows/upstream-release-watch.yml new file mode 100644 index 000000000..c61faae40 --- /dev/null +++ b/.github/workflows/upstream-release-watch.yml @@ -0,0 +1,228 @@ +name: Upstream Release Watch + +# Keeps the fork level with DataDog's released version. +# +# Upstream publishes a v GitHub release. Two things have to be true for us to +# match it: main must contain upstream v, and Tero-Datadog-Extension- must +# be published. This job checks both once a day and moves whichever one is +# behind: +# +# main is behind -> open a PR merging upstream v +# main is level, unreleased -> call release-extension.yml for v +# both level -> nothing to do +# +# The version main contains is recorded in .upstream-version. The merge PR bumps +# it, so releasing only ever happens after a human merged and CI was green. +on: + schedule: + # Daily at 13:00 UTC. Upstream cuts a release roughly monthly, so this is + # about noticing within a day rather than racing. + - cron: "0 13 * * *" + workflow_dispatch: + inputs: + release: + description: "Publish a release when main is level with upstream but unreleased" + required: false + default: true + type: boolean + +permissions: + contents: write + pull-requests: write + issues: write + id-token: write + +jobs: + check: + name: Check upstream release + runs-on: ubuntu-22.04 + outputs: + upstream_version: ${{ steps.compare.outputs.upstream_version }} + merged_version: ${{ steps.compare.outputs.merged_version }} + needs_merge: ${{ steps.compare.outputs.needs_merge }} + needs_release: ${{ steps.compare.outputs.needs_release }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.AWS_ROLE_ARN }} + aws-region: us-east-1 + + - name: Compare upstream release with our state + id: compare + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + + UPSTREAM=$(gh release view \ + --repo DataDog/datadog-lambda-extension \ + --json tagName --jq '.tagName' | sed 's/^v//') + if [[ ! "$UPSTREAM" =~ ^[0-9]+$ ]]; then + echo "::error::Upstream latest release '$UPSTREAM' is not a plain version" + exit 1 + fi + + MERGED=$(tr -d '[:space:]' < .upstream-version) + echo "Upstream released: v$UPSTREAM | main contains: v$MERGED" + + NEEDS_MERGE=false + NEEDS_RELEASE=false + + if [ "$MERGED" -lt "$UPSTREAM" ]; then + NEEDS_MERGE=true + else + # main is level. Has it shipped? us-east-1 is the reference region. + # ponytail: one region, not all nine — a partial multi-region failure + # shows up in the release run itself. Check every region here if that + # stops being true. + VERSIONS=$(aws lambda list-layer-versions \ + --layer-name "Tero-Datadog-Extension-${UPSTREAM}" \ + --query 'length(LayerVersions)' \ + --output text 2>/dev/null || echo "0") + if [ -z "$VERSIONS" ] || [ "$VERSIONS" = "None" ]; then + VERSIONS=0 + fi + if [ "$VERSIONS" -eq 0 ]; then + NEEDS_RELEASE=true + fi + fi + + { + echo "upstream_version=$UPSTREAM" + echo "merged_version=$MERGED" + echo "needs_merge=$NEEDS_MERGE" + echo "needs_release=$NEEDS_RELEASE" + } >> "$GITHUB_OUTPUT" + + { + echo "## Upstream release watch" + echo "" + echo "| | Version |" + echo "|---|---|" + echo "| DataDog released | v$UPSTREAM |" + echo "| main contains | v$MERGED |" + echo "" + if [ "$NEEDS_MERGE" = true ]; then + echo "main is behind — opening a merge PR." + elif [ "$NEEDS_RELEASE" = true ]; then + echo "main is level with upstream but v$UPSTREAM is not published." + else + echo "In parity. Nothing to do." + fi + } >> "$GITHUB_STEP_SUMMARY" + + merge: + name: Open upstream merge PR + runs-on: ubuntu-22.04 + needs: check + if: needs.check.outputs.needs_merge == 'true' + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Merge upstream release + id: merge + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + UPSTREAM_VERSION: ${{ needs.check.outputs.upstream_version }} + run: | + set -euo pipefail + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + BRANCH="chore/upstream-v${UPSTREAM_VERSION}" + if git ls-remote --exit-code --heads origin "$BRANCH" >/dev/null 2>&1; then + echo "Branch $BRANCH already exists; a merge is already in flight." + exit 0 + fi + + git remote add upstream https://github.com/DataDog/datadog-lambda-extension.git + git fetch --tags upstream "v${UPSTREAM_VERSION}" + git checkout -b "$BRANCH" + + if ! git merge --no-edit "v${UPSTREAM_VERSION}"; then + CONFLICTS=$(git diff --name-only --diff-filter=U) + git merge --abort + echo "conflicts<> "$GITHUB_OUTPUT" + echo "$CONFLICTS" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + echo "conflicted=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + echo "$UPSTREAM_VERSION" > .upstream-version + git commit -am "chore: record upstream v${UPSTREAM_VERSION}" --allow-empty + git push origin "$BRANCH" + echo "conflicted=false" >> "$GITHUB_OUTPUT" + + - name: Open pull request + if: steps.merge.outputs.conflicted == 'false' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + UPSTREAM_VERSION: ${{ needs.check.outputs.upstream_version }} + run: | + set -euo pipefail + gh pr create \ + --base main \ + --head "chore/upstream-v${UPSTREAM_VERSION}" \ + --title "chore: merge upstream v${UPSTREAM_VERSION}" \ + --body "$(printf '%s\n' \ + "## Overview" \ + "" \ + "Merges DataDog upstream \`v${UPSTREAM_VERSION}\` and records it in \`.upstream-version\`." \ + "Opened automatically by Upstream Release Watch." \ + "" \ + "## Testing" \ + "" \ + "CI must be green before merge. Once this lands, the daily watch publishes" \ + "\`Tero-Datadog-Extension-${UPSTREAM_VERSION}\` to every default region." )" + + # A conflicted merge needs judgement — stale lock entries and API breakage + # that only clippy catches. Report it and let a human drive. + - name: Report conflicts + if: steps.merge.outputs.conflicted == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + UPSTREAM_VERSION: ${{ needs.check.outputs.upstream_version }} + CONFLICTS: ${{ steps.merge.outputs.conflicts }} + run: | + set -euo pipefail + TITLE="Upstream v${UPSTREAM_VERSION} merge has conflicts" + if gh issue list --state open --search "$TITLE in:title" --json title \ + --jq '.[].title' | grep -qxF "$TITLE"; then + echo "Issue already open." + exit 0 + fi + gh issue create --title "$TITLE" --body "$(printf '%s\n' \ + "Merging upstream \`v${UPSTREAM_VERSION}\` into main conflicts, so no PR was opened." \ + "" \ + "Conflicted files:" \ + "" \ + '```' \ + "$CONFLICTS" \ + '```' \ + "" \ + "Resolve locally, then set \`.upstream-version\` to ${UPSTREAM_VERSION}:" \ + "" \ + '```bash' \ + "git remote add upstream https://github.com/DataDog/datadog-lambda-extension.git" \ + "git fetch --tags upstream v${UPSTREAM_VERSION}" \ + "git checkout -b chore/upstream-v${UPSTREAM_VERSION} && git merge v${UPSTREAM_VERSION}" \ + '```' )" + + release: + name: Release v${{ needs.check.outputs.upstream_version }} + needs: check + if: | + needs.check.outputs.needs_release == 'true' + && (github.event_name == 'schedule' || inputs.release) + uses: ./.github/workflows/release-extension.yml + with: + version: ${{ needs.check.outputs.upstream_version }} + secrets: inherit diff --git a/.upstream-version b/.upstream-version new file mode 100644 index 000000000..3ad5abd03 --- /dev/null +++ b/.upstream-version @@ -0,0 +1 @@ +99 diff --git a/Releasing.md b/Releasing.md index 17bf58dde..beea2108d 100644 --- a/Releasing.md +++ b/Releasing.md @@ -41,8 +41,12 @@ aws iam create-open-id-connect-provider \ "Statement": [ { "Effect": "Allow", - "Action": ["lambda:PublishLayerVersion", "lambda:ListLayerVersions"], - "Resource": "arn:aws:lambda:*:YOUR_ACCOUNT_ID:layer:Datadog-Extension*" + "Action": [ + "lambda:PublishLayerVersion", + "lambda:ListLayerVersions", + "lambda:AddLayerVersionPermission" + ], + "Resource": "arn:aws:lambda:*:YOUR_ACCOUNT_ID:layer:Tero-Datadog-Extension*" }, { "Effect": "Allow", @@ -102,48 +106,103 @@ Replace: - **Name:** `AWS_ROLE_ARN` - **Value:** Your role ARN -## Releasing via GitHub Actions +## Versioning -### Option 1: Manual Release (Recommended) +Releases track DataDog's upstream releases. Upstream publishes `v` tags, so a +Tero release named for upstream v119 contains upstream v119. -1. Go to **Actions → Release Lambda Extension → Run workflow** +An AWS layer version is a single integer, and AWS only ever appends to it — you +cannot ask for version 119. So the upstream version goes in the layer **name**, +and the layer **version** integer is the patch number: -2. Configure the release options: +| Release | Layer name | Layer version | +| ------- | ------------------------------- | ------------- | +| v119 | `Tero-Datadog-Extension-119` | 1 | +| v119.2 | `Tero-Datadog-Extension-119` | 2 | +| v120 | `Tero-Datadog-Extension-120` | 1 | - | Option | Description | Example | - | --------------- | ---------------------------------------------------- | ------------------------------- | - | `version` | Version number (auto-increments if empty) | `100` | - | `layer_suffix` | Suffix to differentiate from official Datadog layers | `-Tero` | - | `regions` | Comma-separated AWS regions | `us-east-1,us-west-2,eu-west-1` | - | `architectures` | Which architectures to build | `amd64,arm64` | - | `fips` | Include FIPS-compliant builds | `false` | - | `dry_run` | Build without publishing to AWS | `false` | +This keeps the upstream number exact in every region without publishing filler +versions to advance a counter, and it gives patches somewhere to live. A new +region starts at patch 1 for whatever upstream versions you publish there, so +patch numbers can differ per region; the upstream number never does. -3. Click **Run workflow** +## Staying level with upstream -4. After completion, find the Layer ARNs in the workflow summary +`.github/workflows/upstream-release-watch.yml` runs daily at 13:00 UTC and can +be triggered by hand. It compares DataDog's latest release with two things: the +version `main` contains (recorded in `.upstream-version`) and whether +`Tero-Datadog-Extension-` is published. + +| State | What the watch does | +| ----------------------------------------- | ------------------------------------ | +| `main` is behind upstream | Opens a PR merging upstream `v` | +| `main` is level, `v` not published | Publishes `v` to all regions | +| Both level | Nothing | + +A release therefore only happens after a human merged the upstream PR and CI was +green. When the merge conflicts the watch opens an issue instead of a PR, because +resolving an upstream merge needs judgement — stale lock entries and API changes +that only clippy catches. -### Option 2: Tag-based Release +To check parity without publishing, run it by hand with `release` unticked. -Push a version tag to trigger an automatic release: +`.upstream-version` is the record of what `main` contains. The merge PR bumps it; +set it by hand if you merge upstream without the watch. + +## Releasing via GitHub Actions + +### Option 1: Tag-based release (recommended) ```bash -git tag v100 -git push origin v100 +git tag v119 # first release of upstream v119 +git push origin v119 + +git tag v119.2 # patch on top of upstream v119 +git push origin v119.2 ``` -This publishes to the default region (us-east-1) with both architectures. +A bare `v119` publishes whatever patch comes next. A `v119.2` tag asserts the +release lands on patch 2 and fails **before** publishing if it would not — a +layer version cannot be renumbered after the fact. + +Tag-based releases publish both architectures to every region in +`DEFAULT_REGIONS` (see `.github/workflows/release-extension.yml`): all four US +regions and the five EU regions that AWS enables by default. + +Opt-in regions (eu-south-1, eu-south-2, eu-central-2) are excluded, because +publishing to a region the account has not enabled fails the job. Add them to +`DEFAULT_REGIONS` once they are enabled. + +### Option 2: Manual release + +1. Go to **Actions → Release Lambda Extension → Run workflow** + +2. Configure the release options: + + | Option | Description | Example | + | --------------- | -------------------------------------- | ------------------------------- | + | `version` | Upstream version, patch optional | `119` or `119.2` | + | `regions` | Comma-separated AWS regions | `us-east-1,us-west-2,eu-west-1` | + | `architectures` | Which architectures to build | `amd64,arm64` | + | `dry_run` | Build without publishing to AWS | `false` | + +3. Click **Run workflow** + +4. After completion, find the Layer ARNs in the workflow summary + +Manual runs append `-dev` to the layer name, so they never overwrite a real +release. ## Layer Naming Convention -The workflow creates layers with this naming pattern: +| Architecture | Trigger | Layer Name | +| ------------ | ---------------- | ------------------------------------ | +| amd64 | tag | `Tero-Datadog-Extension-119` | +| arm64 | tag | `Tero-Datadog-Extension-119-ARM` | +| amd64 | manual | `Tero-Datadog-Extension-119-dev` | +| arm64 | manual | `Tero-Datadog-Extension-119-ARM-dev` | -| Architecture | FIPS | Suffix | Layer Name | -| ------------ | ---- | ------- | --------------------------------- | -| amd64 | No | `-Tero` | `Datadog-Extension-Tero` | -| arm64 | No | `-Tero` | `Datadog-Extension-ARM-Tero` | -| amd64 | Yes | `-Tero` | `Datadog-Extension-FIPS-Tero` | -| arm64 | Yes | `-Tero` | `Datadog-Extension-ARM-FIPS-Tero` | +Substitute the upstream version you are releasing for `119`. ## Using the Published Layer @@ -158,7 +217,7 @@ arn:aws:lambda:REGION:ACCOUNT_ID:layer:LAYER_NAME:VERSION Example: ``` -arn:aws:lambda:us-east-1:123456789012:layer:Datadog-Extension-ARM-Tero:1 +arn:aws:lambda:us-east-1:123456789012:layer:Tero-Datadog-Extension-119-ARM:1 ``` ### Update a Lambda function @@ -168,7 +227,7 @@ arn:aws:lambda:us-east-1:123456789012:layer:Datadog-Extension-ARM-Tero:1 ```bash aws lambda update-function-configuration \ --function-name my-function \ - --layers "arn:aws:lambda:us-east-1:123456789012:layer:Datadog-Extension-ARM-Tero:1" + --layers "arn:aws:lambda:us-east-1:123456789012:layer:Tero-Datadog-Extension-119-ARM:1" ``` **Terraform:** @@ -178,7 +237,7 @@ resource "aws_lambda_function" "example" { # ... other configuration ... layers = [ - "arn:aws:lambda:us-east-1:123456789012:layer:Datadog-Extension-ARM-Tero:1" + "arn:aws:lambda:us-east-1:123456789012:layer:Tero-Datadog-Extension-119-ARM:1" ] } ``` @@ -190,7 +249,7 @@ MyFunction: Type: AWS::Serverless::Function Properties: Layers: - - arn:aws:lambda:us-east-1:123456789012:layer:Datadog-Extension-ARM-Tero:1 + - arn:aws:lambda:us-east-1:123456789012:layer:Tero-Datadog-Extension-119-ARM:1 ``` **Serverless Framework:** @@ -199,7 +258,7 @@ MyFunction: functions: myFunction: layers: - - arn:aws:lambda:us-east-1:123456789012:layer:Datadog-Extension-ARM-Tero:1 + - arn:aws:lambda:us-east-1:123456789012:layer:Tero-Datadog-Extension-119-ARM:1 ``` ## Local Build and Manual Publish @@ -226,7 +285,7 @@ export AWS_REGION=us-east-1 # Publish ARM64 layer aws lambda publish-layer-version \ - --layer-name "Datadog-Extension-ARM-Tero" \ + --layer-name "Tero-Datadog-Extension-119-ARM" \ --description "Tero fork of Datadog Lambda Extension" \ --zip-file "fileb://.layers/datadog_extension-arm64.zip" \ --compatible-architectures arm64 \ @@ -234,7 +293,7 @@ aws lambda publish-layer-version \ # Publish AMD64 layer aws lambda publish-layer-version \ - --layer-name "Datadog-Extension-Tero" \ + --layer-name "Tero-Datadog-Extension-119" \ --description "Tero fork of Datadog Lambda Extension" \ --zip-file "fileb://.layers/datadog_extension-amd64.zip" \ --compatible-architectures x86_64 \ @@ -253,12 +312,15 @@ and update the SourceForge URL if needed. Ensure you're using `policy-rs` version 1.1.1 or later, which includes the ARM64 compatibility fix. -### Layer version already exists +### Release fails with "would publish patch N, but M was requested" -The workflow skips publishing if the target version already exists. Either: +A `v119.2` tag asserts the release lands on patch 2. The check runs before +publishing, because a layer version cannot be renumbered once it exists. The +error reports the patch that would actually be published — re-tag with that +number, or drop the patch (`v119`) to accept whatever comes next. -- Let the version auto-increment (leave version field empty) -- Specify a higher version number +Patch numbers are per region, so a region added later sits behind the others. +The upstream number in the layer name is unaffected. ### Permission denied when publishing @@ -273,7 +335,8 @@ and the trust policy allows your repository. ## Multi-Region Deployment -To deploy to multiple regions, specify them comma-separated: +Releases go to every region in `DEFAULT_REGIONS` by default. To publish to a +different set, pass them comma-separated to a manual run: ``` us-east-1,us-west-2,eu-west-1,ap-southeast-1 @@ -281,3 +344,6 @@ us-east-1,us-west-2,eu-west-1,ap-southeast-1 Each region will get its own copy of the layer. Lambda functions must use a layer from the same region they're deployed in. + +Adding a region needs no catch-up work: the upstream version lives in the layer +name, so a new region publishes `Tero-Datadog-Extension-119:1` directly. diff --git a/scripts/parse_release_version.sh b/scripts/parse_release_version.sh new file mode 100755 index 000000000..4f0b635da --- /dev/null +++ b/scripts/parse_release_version.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# +# Parse a release version into its upstream and patch parts. +# +# Versioning tracks DataDog's upstream releases: the upstream version goes in the +# layer name (Tero-Datadog-Extension-119) and the AWS layer version integer is +# the patch. See .github/workflows/release-extension.yml. +# +# 119 -> upstream 119, patch "" (take the next version AWS assigns) +# 119.2 -> upstream 119, patch 2 (assert it lands on 2 before publishing) +# +# A leading "v" is optional, so a tag ref can be passed straight through. +# +# Usage: parse_release_version.sh v119.2 # prints "119 2" +# parse_release_version.sh --self-test +set -euo pipefail + +parse_release_version() { + local raw="${1#v}" + + if [[ ! "$raw" =~ ^([0-9]+)(\.([0-9]+))?$ ]]; then + echo "version '$1' is not or ." >&2 + return 1 + fi + + local upstream="${BASH_REMATCH[1]}" patch="${BASH_REMATCH[3]:-}" + + # AWS numbers layer versions from 1, so patch 0 can never be published. + if [ -n "$patch" ] && [ "$patch" -lt 1 ]; then + echo "version '$1' has patch $patch; patches start at 1" >&2 + return 1 + fi + + echo "$upstream $patch" +} + +self_test() { + local failures=0 + + check_ok() { # $1 = input, $2 = expected output + local got + if ! got="$(parse_release_version "$1" 2>/dev/null)"; then + echo "FAIL: '$1' was rejected, expected '$2'" + failures=$((failures + 1)) + return + fi + if [ "$got" != "$2" ]; then + echo "FAIL: '$1' gave '$got', expected '$2'" + failures=$((failures + 1)) + fi + } + + check_rejected() { # $1 = input + if parse_release_version "$1" >/dev/null 2>&1; then + echo "FAIL: '$1' was accepted, expected rejection" + failures=$((failures + 1)) + fi + } + + check_ok v119 "119 " + check_ok 119 "119 " + check_ok v119.2 "119 2" + check_ok 119.10 "119 10" + check_ok v100.1 "100 1" + + check_rejected v119.0 # patches start at 1 + check_rejected v119. + check_rejected v1.2.3 + check_rejected vabc + check_rejected "" + check_rejected v-1 + check_rejected "119 2" + + if [ "$failures" -ne 0 ]; then + echo "$failures check(s) failed" + return 1 + fi + echo "all checks passed" +} + +if [ "${1:-}" = "--self-test" ]; then + self_test +else + parse_release_version "${1:-}" +fi