From b5d9eb509a699b65e72ec0d66a902ffa4cde9557 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 12:55:18 -0400 Subject: [PATCH 01/20] [FEATURE](ci) Branching strategy Phase 3 - PyPI and CodeArtifact publish workflows Adds the publish side of the versioning pipeline built in Phase 2.B: - main-publish.yaml: on push to main, diffs changed packages/** files (via the new detect-affected-packages action) to find packages touched without a version bump, stamps a .postN+main. build via compute-version, and publishes to CodeArtifact. Also runs as a build-only smoke test on PRs touching the composite actions or itself, replacing compute-versions-dry-run.yaml. - elease-publish.yaml: on elease: published, parses -v from the tag, builds, and publishes to PyPI via `pypa/gh-action-pypi-publish` (OIDC trusted publishing + attestations), gated by the pypi-release environment's required reviewers. - elease-trigger.yaml: releases now get created with an overture-release-publisher app installation token instead of GITHUB_TOKEN, since GITHUB_TOKEN-created releases don't fire elease: published for other workflows to pick up. App provisioning is tracked separately in #637 and has to happen before this path works. Deletes compute-versions-dry-run.yaml and publish-python-packages.yaml, both superseded by the two workflows above. docs/versioning.md gets a workflow-name reference table and the pypi-release approval-gate mechanics. Still open, all external/manual, tracked on #509: - provisioning overture-release-publisher (#637) - PyPI Trusted Publisher config per package - pypi-release environment + reviewers - p3-dev-builds-ca, blocked on ops-team#299 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-affected-packages/action.yml | 52 ++++++ .../detect_affected_packages.py | 88 +++++++++ .../workflows/compute-versions-dry-run.yaml | 130 ------------- .github/workflows/main-publish.yaml | 173 ++++++++++++++++++ .../workflows/publish-python-packages.yaml | 122 ------------ .github/workflows/release-publish.yaml | 118 ++++++++++++ .github/workflows/release-trigger.yaml | 25 ++- docs/versioning.md | 21 ++- 8 files changed, 469 insertions(+), 260 deletions(-) create mode 100644 .github/actions/detect-affected-packages/action.yml create mode 100644 .github/actions/detect-affected-packages/detect_affected_packages.py delete mode 100644 .github/workflows/compute-versions-dry-run.yaml create mode 100644 .github/workflows/main-publish.yaml delete mode 100644 .github/workflows/publish-python-packages.yaml create mode 100644 .github/workflows/release-publish.yaml diff --git a/.github/actions/detect-affected-packages/action.yml b/.github/actions/detect-affected-packages/action.yml new file mode 100644 index 000000000..2cf7761eb --- /dev/null +++ b/.github/actions/detect-affected-packages/action.yml @@ -0,0 +1,52 @@ +name: Detect affected packages +description: > + Detects packages affected by a no-bump push to main, i.e. any file changed + under packages// between two commits, excluding packages whose + pyproject.toml version was also bumped in the same range (those release via + release-trigger instead; see docs/versioning.md) and packages removed in the + range (nothing left to build). + + Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` is + reachable. + +inputs: + before: + description: The base commit SHA to compare against (e.g. github.event.before). + required: true + after: + description: The head commit SHA to compare to. Defaults to the checked-out HEAD. + required: false + default: HEAD + +outputs: + count: + description: Number of affected packages. + value: ${{ steps.filter.outputs.count }} + packages: + description: > + JSON array of affected package directory names, e.g. + ["overture-schema-common"]. Suitable as a matrix input. + value: ${{ steps.filter.outputs.packages }} + +runs: + using: composite + steps: + - name: Diff package versions + id: diff + shell: bash + env: + BEFORE: ${{ inputs.before }} + AFTER: ${{ inputs.after }} + run: | + python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ + > "${RUNNER_TEMP}/package-version-diff.json" + + - name: Detect affected packages + id: filter + shell: bash + env: + BEFORE: ${{ inputs.before }} + AFTER: ${{ inputs.after }} + run: | + python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" "$BEFORE" "$AFTER" \ + < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-affected-packages/detect_affected_packages.py b/.github/actions/detect-affected-packages/detect_affected_packages.py new file mode 100644 index 000000000..9e679ebad --- /dev/null +++ b/.github/actions/detect-affected-packages/detect_affected_packages.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +""" +Detect packages affected by a no-bump push to main. + +Composes `package_versions.py diff`'s version-diff JSON (read from stdin) with +a file-level `git diff` to find packages whose directory changed without also +changing their pyproject.toml version. Those need an internal `.postN` build +(see docs/versioning.md). Packages with a version bump in the same range are +excluded because they release via `release-trigger` instead, and removed +packages are excluded because there is nothing left to build. + +Run from the repository root, piping `package_versions.py diff`'s output in: + + python3 package_versions.py diff BEFORE AFTER \\ + | python3 detect_affected_packages.py BEFORE AFTER + +Prints `$GITHUB_OUTPUT` lines on stdout (progress goes to stderr): + + count= Number of affected packages. + packages= JSON array of affected package directory names, e.g. + ["overture-schema-common"]. Suitable as a matrix input. + +Exit status: + 0 Success (including the no-affected case). +""" + +import json +import subprocess +import sys + +PACKAGES_DIR = "packages" + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], capture_output=True, check=True) + return result.stdout.decode("utf-8") + + +def changed_package_dirs(before: str, after: str) -> set[str]: + """Package directory names with any file change under packages//.""" + diff = git("diff", "--name-only", f"{before}..{after}", "--", f"{PACKAGES_DIR}/") + names = set() + for line in diff.splitlines(): + parts = line.split("/", 2) + if len(parts) >= 2: + names.add(parts[1]) + return names + + +def main() -> None: + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} BEFORE_COMMIT AFTER_COMMIT", file=sys.stderr) + sys.exit(1) + before, after = sys.argv[1], sys.argv[2] + + version_diff = json.load(sys.stdin) + bumped = { + change["package"] + for change in version_diff + if change["before"] is not None and change["after"] is not None + } + removed = {change["package"] for change in version_diff if change["after"] is None} + + changed = changed_package_dirs(before, after) + affected = sorted(changed - bumped - removed) + + for package in sorted(changed): + if package in removed: + print(f"{package}: removed. Skipping.", file=sys.stderr) + elif package in bumped: + print( + f"{package}: version bumped, handled by release-trigger. " + "Skipping internal build.", + file=sys.stderr, + ) + else: + print(f"{package}: changed, no bump -> internal build.", file=sys.stderr) + + if not affected: + print("No affected packages (no unreleased changes to publish).", file=sys.stderr) + + print(f"count={len(affected)}") + print(f"packages={json.dumps(affected)}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/compute-versions-dry-run.yaml b/.github/workflows/compute-versions-dry-run.yaml deleted file mode 100644 index 5c8ea99f3..000000000 --- a/.github/workflows/compute-versions-dry-run.yaml +++ /dev/null @@ -1,130 +0,0 @@ -name: Compute versions (dry run) - -# Runs on pushes to vnext and main. Computes and logs the version that would -# be published for each affected package — but does not build or publish. -# Also runs (read-only) on PRs that touch the compute-version/code-artifact -# composite actions or this workflow itself, as a smoke test for those. -# Remove this workflow once Phase 3 publish workflows are live (see -# https://github.com/OvertureMaps/schema/issues/509). - -on: - # Real usage: logs the version that would be published for the actual - # branch context. - push: - branches: [main, vnext] - paths: - - '**/pyproject.toml' - # Test usage: smoke-tests the compute-version/code-artifact composite - # actions (and this workflow) against a placeholder context on PRs that - # touch them, so wiring regressions surface before merge. - pull_request: - paths: - - '.github/actions/compute-version/**' - - '.github/actions/code-artifact/**' - - '.github/workflows/compute-versions-dry-run.yaml' - workflow_dispatch: - inputs: - context: - description: "Version context to simulate" - type: choice - options: [vnext, main] - default: vnext - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - discover: - name: Discover context and packages${{ github.event_name == 'pull_request' && ' (test)' || '' }} - if: github.event.repository.full_name == github.repository - runs-on: ubuntu-latest - outputs: - context: ${{ steps.context.outputs.value }} - packages: ${{ steps.packages.outputs.value }} - steps: - - name: Check out code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Determine context - id: context - env: - INPUT_CONTEXT: ${{ inputs.context }} - REF_NAME: ${{ github.ref_name }} - run: | - if [ -n "$INPUT_CONTEXT" ]; then - echo "value=$INPUT_CONTEXT" >> "$GITHUB_OUTPUT" - elif [ "$REF_NAME" = "vnext" ]; then - echo "value=vnext" >> "$GITHUB_OUTPUT" - else - echo "value=main" >> "$GITHUB_OUTPUT" - fi - - - name: Discover packages - id: packages - run: | - packages=$(for pkg_dir in packages/overture-schema*/; do - [ -f "${pkg_dir}/pyproject.toml" ] && basename "$pkg_dir" - done | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "value=${packages}" >> "$GITHUB_OUTPUT" - - compute-versions: - name: Compute version (${{ matrix.package }})${{ github.event_name == 'pull_request' && ' (test)' || '' }} - needs: discover - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # Required for OIDC authentication to AWS - strategy: - fail-fast: false - matrix: - package: ${{ fromJson(needs.discover.outputs.packages) }} - - steps: - - name: Install uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - version: latest - - - name: Check out code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_ReadOnly - role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} - - - name: Get CodeArtifact credentials - id: ca - uses: ./.github/actions/code-artifact - - # Delegates to the same composite action Phase 3 publish workflows will - # use, so the version formula only has one implementation to keep correct. - - name: Compute version - id: compute - uses: ./.github/actions/compute-version - with: - package: ${{ matrix.package }} - context: ${{ needs.discover.outputs.context }} - index_url: ${{ steps.ca.outputs.index_url }} - - - name: Report computed version - env: - PACKAGE: ${{ matrix.package }} - CONTEXT: ${{ needs.discover.outputs.context }} - VERSION: ${{ steps.compute.outputs.version }} - run: | - echo "## Computed version: \`${PACKAGE}\`" >> "$GITHUB_STEP_SUMMARY" - echo "" >> "$GITHUB_STEP_SUMMARY" - echo "Context: \`${CONTEXT}\` " >> "$GITHUB_STEP_SUMMARY" - echo "Version: \`${VERSION}\`" >> "$GITHUB_STEP_SUMMARY" - echo " ${PACKAGE} → ${VERSION}" diff --git a/.github/workflows/main-publish.yaml b/.github/workflows/main-publish.yaml new file mode 100644 index 000000000..8f4779024 --- /dev/null +++ b/.github/workflows/main-publish.yaml @@ -0,0 +1,173 @@ +name: Main publish + +# Runs on every push to main that touches packages/**. For each package whose +# directory changed WITHOUT a version bump, computes an internal build version +# (main context; see .github/actions/compute-version) and publishes it to +# CodeArtifact. Bumped packages release via release-trigger + release-publish +# instead (see docs/versioning.md) -- publishing a .postN for them here too +# would be redundant. +# +# Also runs (read-only) on PRs that touch the compute-version/code-artifact/ +# detect-affected-packages composite actions or this workflow itself, as a +# smoke test for their wiring: the version is computed but nothing is +# published. + +on: + push: + branches: [main] + paths: + - packages/** + pull_request: + paths: + - '.github/actions/compute-version/**' + - '.github/actions/code-artifact/**' + - '.github/actions/detect-affected-packages/**' + - '.github/workflows/main-publish.yaml' + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + # A real push must never be cancelled by a later one: every no-bump push + # needs its own .postN build recorded in CodeArtifact. Only the PR smoke + # test is safe to supersede. + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + detect: + name: Detect affected packages${{ github.event_name == 'pull_request' && ' (test)' || '' }} + if: github.event.repository.full_name == github.repository + runs-on: ubuntu-slim + permissions: + contents: read # Read pyproject.toml/file history to detect affected packages + outputs: + count: ${{ steps.discover.outputs.count || steps.detect.outputs.count }} + packages: ${{ steps.discover.outputs.packages || steps.detect.outputs.packages }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + # A PR touching the composite actions rarely also touches packages/**, + # so the real affected-package diff below would come back empty and + # skip the smoke test entirely. Test every package's wiring instead. + - name: Discover all packages (test) + id: discover + if: github.event_name == 'pull_request' + run: | + packages=$(for pkg_dir in packages/overture-schema*/; do + [ -f "${pkg_dir}/pyproject.toml" ] && basename "$pkg_dir" + done | jq -R -s -c 'split("\n") | map(select(length > 0))') + echo "count=$(echo "$packages" | jq 'length')" >> "$GITHUB_OUTPUT" + echo "packages=${packages}" >> "$GITHUB_OUTPUT" + + - name: Detect affected packages + id: detect + if: github.event_name != 'pull_request' + uses: ./.github/actions/detect-affected-packages + with: + before: ${{ github.event.before }} + + publish: + name: Publish ${{ matrix.package }}${{ github.event_name == 'pull_request' && ' (test)' || '' }} + needs: detect + if: needs.detect.outputs.count != '0' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # Required for OIDC authentication to AWS + strategy: + fail-fast: false # One package's failure must not block sibling publishes + matrix: + package: ${{ fromJSON(needs.detect.outputs.packages) }} + steps: + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: latest + + - name: Check out code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Sync code to make packages visible to Python + run: uv sync --locked --all-packages + + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_Publish + role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} + + - name: Get CodeArtifact credentials + id: ca + uses: ./.github/actions/code-artifact + + # Delegates to the same composite action the PR smoke test exercises, so + # the version formula only has one implementation to keep correct. + - name: Compute version + id: compute + uses: ./.github/actions/compute-version + with: + package: ${{ matrix.package }} + context: main + index_url: ${{ steps.ca.outputs.index_url }} + + # overture-schema-pyspark ships generated validation expressions that are + # not committed to git -- they are regenerated on demand from the + # Pydantic models. `uv build` packages whatever is on disk under the + # module root, so the tree must be generated here before the build, or + # the published wheel ships without its expressions/generated/ modules. + - name: Generate PySpark expressions before build + if: matrix.package == 'overture-schema-pyspark' + run: make generate-pyspark + + - name: Stamp computed version + # Rewrites only the in-memory checkout's pyproject.toml, never + # committed: the released .. stays human-owned, + # this just labels the artifact this job builds and publishes. + env: + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + VERSION: ${{ steps.compute.outputs.version }} # zizmor: ignore[template-injection] + run: uv version "${VERSION}" --package "${PACKAGE}" --frozen + + - name: Build ${{ matrix.package }} ${{ steps.compute.outputs.version }} + env: + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + run: uv build --package "${PACKAGE}" + + - name: Publish ${{ matrix.package }} ${{ steps.compute.outputs.version }} to CodeArtifact + if: github.event_name != 'pull_request' + env: + CA_TOKEN: ${{ steps.ca.outputs.token }} + CA_PUBLISH_URL: ${{ steps.ca.outputs.publish_url }} + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + VERSION: ${{ steps.compute.outputs.version }} # zizmor: ignore[template-injection] + run: | + set -euo pipefail + wheel="dist/${PACKAGE//-/_}-${VERSION}-py3-none-any.whl" + if [ ! -f "$wheel" ]; then + echo " Wheel file [$wheel] not found. Aborting!" + exit 1 + fi + # Guard against a silently empty wheel: if the generate step above + # ever regresses, overture-schema-pyspark would publish with no + # validation expressions. Fail loudly instead of shipping a hollow + # package. + if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ + ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then + echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" + exit 1 + fi + tarball="dist/${PACKAGE//-/_}-${VERSION}.tar.gz" + if [ ! -f "$tarball" ]; then + echo " Source tarball file [$tarball] not found. Aborting!" + exit 1 + fi + uv publish "$wheel" "$tarball" \ + -t "${CA_TOKEN}" \ + --publish-url "${CA_PUBLISH_URL}" diff --git a/.github/workflows/publish-python-packages.yaml b/.github/workflows/publish-python-packages.yaml deleted file mode 100644 index 1c4a9802c..000000000 --- a/.github/workflows/publish-python-packages.yaml +++ /dev/null @@ -1,122 +0,0 @@ -name: Publish Python packages to PyPI - -on: - push: - branches: [main] - paths: - - '**/pyproject.toml' - workflow_dispatch: - inputs: - aws_iam_role_name: - description: The name of the IAM role to assume for accessing CodeArtifact - type: string - required: false - default: GithubActions_Schema_CodeArtifact_Publish - domain: - description: The CodeArtifact domain name - type: string - required: false - default: overture-pypi - repository: - description: The CodeArtifact repository name - type: string - required: false - default: overture - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - check: - name: Check for changes - if: github.event.repository.full_name == github.repository - uses: ./.github/workflows/reusable-check-python-package-versions.yaml - permissions: - contents: read # Required for checkout in reusable workflow - id-token: write # Required for OIDC in reusable workflow to check AWS CodeArtifact - with: - before_commit: ${{ github.event.before }} - after_commit: ${{ github.event.after }} - - publish: - name: Publish - needs: [check] - if: github.event.repository.full_name == github.repository && needs.check.outputs.num_changed_packages > 0 - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write # Required for OIDC authentication to AWS - - strategy: - matrix: - include: ${{ fromJson(needs.check.outputs.changed_packages) }} - steps: - - name: Install uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - version: latest - - - name: Check out code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Sync code to make packages visible to Python - run: uv sync --locked --all-packages - - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_Publish - role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} - - - name: Get CodeArtifact credentials - id: get-code-artifact-params - uses: ./.github/actions/code-artifact - - # overture-schema-pyspark ships generated validation expressions that are - # not committed to git -- they are regenerated on demand from the Pydantic - # models. `uv build` packages whatever is on disk under the module root, so - # the tree must be generated here before the build, or the published wheel - # ships without its `expressions/generated/` modules and validate_model() - # discovers nothing to run. - - name: Generate PySpark expressions before build - if: matrix.package == 'overture-schema-pyspark' - run: make generate-pyspark - - - name: Publish package ${{ matrix.package }} version ${{ matrix.after }} to PyPI - env: - CA_TOKEN: ${{ steps.get-code-artifact-params.outputs.token }} - CA_PUBLISH_URL: ${{ steps.get-code-artifact-params.outputs.publish_url }} - PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] - BEFORE: ${{ matrix.before }} # zizmor: ignore[template-injection] - AFTER: ${{ matrix.after }} # zizmor: ignore[template-injection] - run: | - printf 'Publishing package %s version %s to PyPI (previous version %s)...\n' "$PACKAGE" "$AFTER" "$BEFORE" - uv build --package "$PACKAGE" - wheel="dist/${PACKAGE//-/_}-${AFTER}-py3-none-any.whl" - if [ ! -f "$wheel" ]; then - echo " Wheel file [$wheel] not found. Aborting!" - exit 1 - fi - # Guard against a silently empty wheel: if the generate step above ever - # regresses, overture-schema-pyspark would publish with no validation - # expressions. Fail loudly instead of shipping a hollow package. - if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ - ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then - echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" - exit 1 - fi - tarball="dist/${PACKAGE//-/_}-${AFTER}.tar.gz" - if [ ! -f "$tarball" ]; then - echo " Source tarball file [$tarball] not found. Aborting!" - exit 1 - fi - uv publish "$wheel" "$tarball" \ - -t "${CA_TOKEN}" \ - --publish-url "${CA_PUBLISH_URL}" diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml new file mode 100644 index 000000000..e3da27146 --- /dev/null +++ b/.github/workflows/release-publish.yaml @@ -0,0 +1,118 @@ +name: Release publish + +# Triggered when a GitHub Release is published (created by release-trigger.yaml +# using the overture-release-publisher app token -- see #637 -- so the native +# `release: published` event actually fires; GITHUB_TOKEN would not). +# +# Builds the released package at its released version and publishes it to +# public PyPI via Trusted Publishing (OIDC) + attestations, gated by the +# pypi-release GitHub Environment's required reviewers. +# +# Vanity tags (the legacy bare v series continued by overture-schema +# releases; see docs/versioning.md) never get a second GitHub Release object, +# so this only ever fires once per -v release. +# +# To route approval notifications to Slack instead of the GitHub UI, install +# the GitHub app in the target channel and subscribe it to this repository's +# deployment reviews -- see +# https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-environments-for-deployment +# and https://github.com/integrations/slack#subscribing-to-a-repository. +# No workflow changes needed; approval gating stays entirely in the +# pypi-release Environment. + +on: + release: + types: [published] + +permissions: + contents: read + +concurrency: + # One release event per package version; never cancel an in-flight publish. + group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + cancel-in-progress: false + +jobs: + parse: + name: Parse release tag + if: github.event.repository.full_name == github.repository + runs-on: ubuntu-slim + outputs: + package: ${{ steps.parse.outputs.package }} + version: ${{ steps.parse.outputs.version }} + steps: + - name: Parse package and version from tag + id: parse + env: + TAG: ${{ github.event.release.tag_name }} + run: | + set -euo pipefail + if [[ ! "$TAG" =~ ^(.+)-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + echo "::error::Release tag '${TAG}' is not -v..; nothing to publish." + exit 1 + fi + echo "package=${BASH_REMATCH[1]}" >> "$GITHUB_OUTPUT" + echo "version=${BASH_REMATCH[2]}" >> "$GITHUB_OUTPUT" + + publish: + name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to PyPI + needs: parse + runs-on: ubuntu-latest + environment: pypi-release # Required reviewers gate the PyPI publish step + permissions: + contents: read + id-token: write # Required for PyPI Trusted Publishing (OIDC) + env: + PACKAGE: ${{ needs.parse.outputs.package }} + VERSION: ${{ needs.parse.outputs.version }} + steps: + - name: Install uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + version: latest + enable-cache: false # Publish job builds and ships immediately; no cache to poison downstream runs + + - name: Check out released commit + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.event.release.target_commitish }} + persist-credentials: false + + - name: Sync code to make packages visible to Python + run: uv sync --locked --all-packages + + # overture-schema-pyspark ships generated validation expressions that are + # not committed to git -- they are regenerated on demand from the + # Pydantic models. `uv build` packages whatever is on disk under the + # module root, so the tree must be generated here before the build, or + # the published wheel ships without its expressions/generated/ modules. + - name: Generate PySpark expressions before build + if: env.PACKAGE == 'overture-schema-pyspark' + run: make generate-pyspark + + - name: Build ${{ env.PACKAGE }} ${{ env.VERSION }} + run: uv build --package "${PACKAGE}" + + - name: Verify built version matches the release + run: | + set -euo pipefail + wheel="dist/${PACKAGE//-/_}-${VERSION}-py3-none-any.whl" + if [ ! -f "$wheel" ]; then + echo " Wheel file [$wheel] not found (built version does not match release ${VERSION}). Aborting!" + exit 1 + fi + # Guard against a silently empty wheel: if the generate step above + # ever regresses, overture-schema-pyspark would publish with no + # validation expressions. Fail loudly instead of shipping a hollow + # package to public PyPI. + if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ + ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then + echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" + exit 1 + fi + + - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to PyPI + uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 + with: + packages-dir: dist/ + attestations: true diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index b54b7b75c..f15bba743 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -16,10 +16,10 @@ name: Publish GitHub release # is a release. No-bump merges publish internal `.postN` builds instead (see # .github/actions/compute-version). # -# NOTE: releases are created with GITHUB_TOKEN, which by design does NOT trigger -# further workflow runs. The Phase 3 publish workflow (#509) must therefore be -# triggered by an app/PAT token here or via repository_dispatch, a plain -# `on: release: published` listener will not fire for these releases. +# Releases are created with the overture-release-publisher app's installation +# token, not GITHUB_TOKEN: GITHUB_TOKEN-created releases don't fire downstream +# `release: published` events, which release-publish.yaml needs to pick up +# each release individually. See #637 for provisioning that app. on: push: @@ -62,12 +62,23 @@ jobs: if: needs.detect.outputs.count != '0' runs-on: ubuntu-slim permissions: - contents: write # Required to create releases and their tags + contents: read # App token below carries the actual release-creation grant strategy: fail-fast: false # One package's failure must not block sibling releases matrix: include: ${{ fromJSON(needs.detect.outputs.bumps) }} steps: + - name: Generate app token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + client-id: ${{ secrets.RELEASE_PUBLISHER_APP_ID }} # zizmor: ignore[secrets-outside-env] + private-key: ${{ secrets.RELEASE_PUBLISHER_APP_PEM }} # zizmor: ignore[secrets-outside-env] + # Explicitly scope the app token instead of inheriting all + # installation permissions: creating a release/tag is the only + # write this workflow performs. + permission-contents: write + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false @@ -83,4 +94,6 @@ jobs: # The umbrella package continues the legacy bare tag series as a # vanity tag (no second release); see docs/versioning.md. vanity-tag: ${{ matrix.package == 'overture-schema' && format('v{0}', matrix.version) || '' }} - token: ${{ github.token }} + # App token so the release fires its own `release: published` event + # for release-publish.yaml (GITHUB_TOKEN would not); see #637. + token: ${{ steps.app-token.outputs.token }} diff --git a/docs/versioning.md b/docs/versioning.md index 88a31f995..ef78ea187 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -52,6 +52,16 @@ over the `main` one. `vnext` builds publish only once a separate dev repository exists. Local labels are rejected by public PyPI, which keeps internal builds off the public index by construction. +| Event | Workflow | +|-------|----------| +| Push to `main` | [`main-publish.yaml`](../.github/workflows/main-publish.yaml) detects packages changed without a version bump and publishes their `.postN` build to CodeArtifact. | +| Version bump merged to `main` | [`release-trigger.yaml`](../.github/workflows/release-trigger.yaml) cuts a GitHub Release per bumped package. | +| Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI, gated by the `pypi-release` Environment. | + +`release-trigger` creates releases with the `overture-release-publisher` app's +installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does +not fire the `release: published` event `release-publish` listens for. + ### Workspace dependency floors Intra-repo dependencies follow uv's @@ -172,8 +182,15 @@ changes that package, whether or not it bumps the version. 2. On merge to `main`, `release-trigger` publishes one GitHub Release per bumped package: tag `-v`, notes from that package's `CHANGELOG.md`. -3. Publishing the release starts the PyPI publish, gated by a maintainer - approval. +3. Publishing the release starts the PyPI publish. The `pypi-release` + [GitHub Environment](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-environments-for-deployment)'s + required reviewers gate the publish job; approve or reject from the + workflow run's summary page. To get gate notifications in Slack instead of + polling GitHub, subscribe a channel to this repository's deployment + reviews with the + [GitHub app for Slack](https://github.com/integrations/slack#subscribing-to-a-repository) — + no workflow change needed, the gate itself stays entirely in the + Environment's reviewer list. ```mermaid flowchart LR From 84574c86f190a214296755dbe202431e7b04f5a7 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 13:09:03 -0400 Subject: [PATCH 02/20] [REFACTOR](ci) Use tj-actions/changed-files in detect-affected-packages Replaces the hand-rolled `git diff --name-only` + path-splitting with tj-actions/changed-files' dir_names output, pinned by commit SHA. Fixes a real gap in the old parser: git diff --name-only quotes non-ASCII/unusual filenames, which naive line.split("/") didn't account for. Uses ll_modified_files (ACMRD) rather than ll_changed_files (ACMR) so file deletions still count as a package change, matching the old diff's behavior. detect_affected_packages.py drops its subprocess/git plumbing entirely and just reads the directory list from CHANGED_DIRS; the bump/removed exclusion logic (a three-way set difference) stays in Python rather than jq/bash, that's a different complexity class than this repo's existing jq usage in enforce-change-type-label.yaml. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-affected-packages/action.yml | 19 +++++++-- .../detect_affected_packages.py | 42 +++++-------------- 2 files changed, 27 insertions(+), 34 deletions(-) diff --git a/.github/actions/detect-affected-packages/action.yml b/.github/actions/detect-affected-packages/action.yml index 2cf7761eb..ab8e81c05 100644 --- a/.github/actions/detect-affected-packages/action.yml +++ b/.github/actions/detect-affected-packages/action.yml @@ -41,12 +41,25 @@ runs: python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ > "${RUNNER_TEMP}/package-version-diff.json" + - name: List changed package directories + id: changed-dirs + uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 + with: + base_sha: ${{ inputs.before }} + sha: ${{ inputs.after }} + files: packages/** + dir_names: true + dir_names_max_depth: 2 + matrix: true + - name: Detect affected packages id: filter shell: bash env: - BEFORE: ${{ inputs.before }} - AFTER: ${{ inputs.after }} + # all_modified_files (not all_changed_files) so deletions count too: a + # file removed from a package without a version bump still changes + # the built wheel. + CHANGED_DIRS: ${{ steps.changed-dirs.outputs.all_modified_files }} run: | - python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" "$BEFORE" "$AFTER" \ + python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" \ < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-affected-packages/detect_affected_packages.py b/.github/actions/detect-affected-packages/detect_affected_packages.py index 9e679ebad..825acec93 100644 --- a/.github/actions/detect-affected-packages/detect_affected_packages.py +++ b/.github/actions/detect-affected-packages/detect_affected_packages.py @@ -4,16 +4,18 @@ Detect packages affected by a no-bump push to main. Composes `package_versions.py diff`'s version-diff JSON (read from stdin) with -a file-level `git diff` to find packages whose directory changed without also -changing their pyproject.toml version. Those need an internal `.postN` build -(see docs/versioning.md). Packages with a version bump in the same range are -excluded because they release via `release-trigger` instead, and removed -packages are excluded because there is nothing left to build. +a JSON array of changed package directories (env var `CHANGED_DIRS`, from +tj-actions/changed-files' `dir_names` output) to find packages whose directory +changed without also changing their pyproject.toml version. Those need an +internal `.postN` build (see docs/versioning.md). Packages with a version bump +in the same range are excluded because they release via `release-trigger` +instead, and removed packages are excluded because there is nothing left to +build. Run from the repository root, piping `package_versions.py diff`'s output in: python3 package_versions.py diff BEFORE AFTER \\ - | python3 detect_affected_packages.py BEFORE AFTER + | CHANGED_DIRS='["packages/overture-schema-common"]' python3 detect_affected_packages.py Prints `$GITHUB_OUTPUT` lines on stdout (progress goes to stderr): @@ -26,34 +28,11 @@ """ import json -import subprocess +import os import sys -PACKAGES_DIR = "packages" - - -def git(*args: str) -> str: - result = subprocess.run(["git", *args], capture_output=True, check=True) - return result.stdout.decode("utf-8") - - -def changed_package_dirs(before: str, after: str) -> set[str]: - """Package directory names with any file change under packages//.""" - diff = git("diff", "--name-only", f"{before}..{after}", "--", f"{PACKAGES_DIR}/") - names = set() - for line in diff.splitlines(): - parts = line.split("/", 2) - if len(parts) >= 2: - names.add(parts[1]) - return names - def main() -> None: - if len(sys.argv) != 3: - print(f"Usage: {sys.argv[0]} BEFORE_COMMIT AFTER_COMMIT", file=sys.stderr) - sys.exit(1) - before, after = sys.argv[1], sys.argv[2] - version_diff = json.load(sys.stdin) bumped = { change["package"] @@ -62,7 +41,8 @@ def main() -> None: } removed = {change["package"] for change in version_diff if change["after"] is None} - changed = changed_package_dirs(before, after) + changed_paths = json.loads(os.environ.get("CHANGED_DIRS") or "[]") + changed = {path.split("/", 1)[1] for path in changed_paths} affected = sorted(changed - bumped - removed) for package in sorted(changed): From d5c9f2794e4a0fdbac9cde0514b8de127bf6bbe5 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 13:10:55 -0400 Subject: [PATCH 03/20] [REFACTOR](ci) Reorder detect-affected-packages steps so both Python scripts run back to back The action wasn't wrong, just awkward to read: python, jump to a JS action, jump back to python. Steps 1 (version diff) and 2 (changed dirs) don't depend on each other, only step 3 does, so nothing stops them running adjacent. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-affected-packages/action.yml | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/.github/actions/detect-affected-packages/action.yml b/.github/actions/detect-affected-packages/action.yml index ab8e81c05..789ee46cc 100644 --- a/.github/actions/detect-affected-packages/action.yml +++ b/.github/actions/detect-affected-packages/action.yml @@ -31,16 +31,6 @@ outputs: runs: using: composite steps: - - name: Diff package versions - id: diff - shell: bash - env: - BEFORE: ${{ inputs.before }} - AFTER: ${{ inputs.after }} - run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ - > "${RUNNER_TEMP}/package-version-diff.json" - - name: List changed package directories id: changed-dirs uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6 @@ -52,6 +42,18 @@ runs: dir_names_max_depth: 2 matrix: true + # Independent of the step above (neither reads the other's output); runs + # after it only so both Python steps land back to back. + - name: Diff package versions + id: diff + shell: bash + env: + BEFORE: ${{ inputs.before }} + AFTER: ${{ inputs.after }} + run: | + python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ + > "${RUNNER_TEMP}/package-version-diff.json" + - name: Detect affected packages id: filter shell: bash From e975d12f32c86f2bd591f424ea050a7ef9666915 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 13:19:40 -0400 Subject: [PATCH 04/20] [REFACTOR](ci) Extract diff-package-versions composite action package_versions.py diff was inlined at three call sites (detect-version-bumps, detect-affected-packages, and eusable-check-python-package-versions.yaml), each reimplementing the temp-file-then-$GITHUB_OUTPUT plumbing slightly differently (one even wrote to a different temp path and used heredoc-style output). diff-package-versions is now the one place that knows how to run the script and expose it as count/diff outputs. The three consumers just pipe steps.diff.outputs.diff into their own filter logic: detect_version_bumps.py and detect_affected_packages.py are unchanged, they already read JSON from stdin. eusable-check-python-package-versions.yaml's CodeArtifact existence check now reads the diff from an env var instead of a temp file; its changed_packages output is compact JSON now instead of pretty-printed, no consumer depends on the formatting. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-affected-packages/action.yml | 17 +++---- .../actions/detect-version-bumps/action.yml | 14 +++--- .../actions/diff-package-versions/action.yml | 46 +++++++++++++++++++ ...eusable-check-python-package-versions.yaml | 23 ++++++---- 4 files changed, 72 insertions(+), 28 deletions(-) create mode 100644 .github/actions/diff-package-versions/action.yml diff --git a/.github/actions/detect-affected-packages/action.yml b/.github/actions/detect-affected-packages/action.yml index 789ee46cc..a99fdade7 100644 --- a/.github/actions/detect-affected-packages/action.yml +++ b/.github/actions/detect-affected-packages/action.yml @@ -43,16 +43,13 @@ runs: matrix: true # Independent of the step above (neither reads the other's output); runs - # after it only so both Python steps land back to back. + # after it only so both feed the final filter step back to back. - name: Diff package versions id: diff - shell: bash - env: - BEFORE: ${{ inputs.before }} - AFTER: ${{ inputs.after }} - run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ - > "${RUNNER_TEMP}/package-version-diff.json" + uses: ./.github/actions/diff-package-versions + with: + before: ${{ inputs.before }} + after: ${{ inputs.after }} - name: Detect affected packages id: filter @@ -62,6 +59,6 @@ runs: # file removed from a package without a version bump still changes # the built wheel. CHANGED_DIRS: ${{ steps.changed-dirs.outputs.all_modified_files }} + DIFF: ${{ steps.diff.outputs.diff }} run: | - python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" \ - < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT" + echo "$DIFF" | python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-version-bumps/action.yml b/.github/actions/detect-version-bumps/action.yml index 22cab9ab0..eed3460cc 100644 --- a/.github/actions/detect-version-bumps/action.yml +++ b/.github/actions/detect-version-bumps/action.yml @@ -31,16 +31,14 @@ runs: steps: - name: Diff package versions id: diff - shell: bash - env: - BEFORE: ${{ inputs.before }} - run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" HEAD \ - > "${RUNNER_TEMP}/package-version-diff.json" + uses: ./.github/actions/diff-package-versions + with: + before: ${{ inputs.before }} - name: Filter to releasable bumps id: filter shell: bash + env: + DIFF: ${{ steps.diff.outputs.diff }} run: | - python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" \ - < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT" + echo "$DIFF" | python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/diff-package-versions/action.yml b/.github/actions/diff-package-versions/action.yml new file mode 100644 index 000000000..5b8a79684 --- /dev/null +++ b/.github/actions/diff-package-versions/action.yml @@ -0,0 +1,46 @@ +name: Diff package versions +description: > + Diffs packages/*/pyproject.toml versions between two commits (see + package_versions.py), reading blobs directly from git. Enforces the + major-bump cascade: a package whose direct workspace dependency takes a + major bump must take one itself; the action fails otherwise. + + Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` + is reachable. + +inputs: + before: + description: The base commit SHA to compare against (e.g. github.event.before). + required: true + after: + description: The head commit SHA to compare to. Defaults to the checked-out HEAD. + required: false + default: HEAD + +outputs: + count: + description: Number of packages with a version change (added, removed, or bumped). + value: ${{ steps.diff.outputs.count }} + diff: + description: > + JSON array of {"package", "before", "after"} objects, topologically + sorted (dependencies before dependents). `before`/`after` are null when + the package didn't exist at that commit. + value: ${{ steps.diff.outputs.diff }} + +runs: + using: composite + steps: + - name: Diff package versions + id: diff + shell: bash + env: + BEFORE: ${{ inputs.before }} + AFTER: ${{ inputs.after }} + run: | + python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ + > "${RUNNER_TEMP}/package-version-diff.json" + { + echo "count=$(jq 'length' "${RUNNER_TEMP}/package-version-diff.json")" + echo "diff=$(jq -c . "${RUNNER_TEMP}/package-version-diff.json")" + } >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/reusable-check-python-package-versions.yaml b/.github/workflows/reusable-check-python-package-versions.yaml index 6f2d68258..9a1be66aa 100644 --- a/.github/workflows/reusable-check-python-package-versions.yaml +++ b/.github/workflows/reusable-check-python-package-versions.yaml @@ -78,19 +78,21 @@ jobs: persist-credentials: false - name: Diff package versions + id: diff + uses: ./.github/actions/diff-package-versions + with: + before: ${{ inputs.before_commit }} + after: ${{ inputs.after_commit }} + + - name: Save changed packages id: save-changes env: - BEFORE: ${{ inputs.before_commit }} - AFTER: ${{ inputs.after_commit }} + DIFF: ${{ steps.diff.outputs.diff }} + COUNT: ${{ steps.diff.outputs.count }} run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ - > /tmp/package-version-diff.json - cat /tmp/package-version-diff.json { - echo 'changed_packages<> "$GITHUB_OUTPUT" - name: Configure AWS credentials @@ -115,9 +117,10 @@ jobs: if: steps.save-changes.outputs.num_changed_packages > 0 env: INDEX_URL: ${{ steps.get-code-artifact-index-url.outputs.index_url }} + DIFF: ${{ steps.diff.outputs.diff }} # zizmor: ignore[template-injection] run: | - jq -c '.[]' /tmp/package-version-diff.json | while read -r entry; do + jq -c '.[]' <<< "$DIFF" | while read -r entry; do package=$(echo "$entry" | jq -r '.package') after=$(echo "$entry" | jq -r '.after') if [ "$after" = "null" ]; then From 99b1fd1ef97adb84c2164dea5883239d4106f618 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 13:24:06 -0400 Subject: [PATCH 05/20] [REFACTOR](ci) Move package_versions.py into diff-package-versions Last remaining dangling script: package_versions.py lived in .github/workflows/scripts/ and was reached into by relative path. Every other action already keeps its script(s) alongside its own action.yml (detect-version-bumps, detect-affected-packages, create-package-release); this was the one holdout, and after the last refactor diff-package-versions is its only remaining caller anyway. Moved it in, switched the reference to ${GITHUB_ACTION_PATH}, and removed the now-empty scripts/ directory. No behavior change: git commands inside the script still run with the repo root as CWD, only the path used to invoke it changed. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/actions/diff-package-versions/action.yml | 2 +- .../diff-package-versions}/package_versions.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) rename .github/{workflows/scripts => actions/diff-package-versions}/package_versions.py (96%) diff --git a/.github/actions/diff-package-versions/action.yml b/.github/actions/diff-package-versions/action.yml index 5b8a79684..8ad20d8f9 100644 --- a/.github/actions/diff-package-versions/action.yml +++ b/.github/actions/diff-package-versions/action.yml @@ -38,7 +38,7 @@ runs: BEFORE: ${{ inputs.before }} AFTER: ${{ inputs.after }} run: | - python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ + python3 "${GITHUB_ACTION_PATH}/package_versions.py" diff "$BEFORE" "$AFTER" \ > "${RUNNER_TEMP}/package-version-diff.json" { echo "count=$(jq 'length' "${RUNNER_TEMP}/package-version-diff.json")" diff --git a/.github/workflows/scripts/package_versions.py b/.github/actions/diff-package-versions/package_versions.py similarity index 96% rename from .github/workflows/scripts/package_versions.py rename to .github/actions/diff-package-versions/package_versions.py index c13e7a6f2..2064f00b7 100644 --- a/.github/workflows/scripts/package_versions.py +++ b/.github/actions/diff-package-versions/package_versions.py @@ -3,9 +3,11 @@ """ Diff per-package versions between two git commits. -Run from the repository root: +Run with the repository root as the working directory (git commands need it +as CWD); the script itself can live anywhere, e.g. the `diff-package-versions` +action invokes it via `$GITHUB_ACTION_PATH`: - python3 package_versions.py diff + python3 path/to/package_versions.py diff Reads each `packages/*/pyproject.toml` blob directly from git at both commits (no checkout switching, no environment sync) and prints the packages whose From 357b1ca0b3829aeeb43c51f209d407f337827a39 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 13:34:42 -0400 Subject: [PATCH 06/20] [REFACTOR](docs) Polish CONTRIBUTING.md for the finished branching/release flow Phase 3 was the last phase (#509); Phase 4 got folded into it instead of staying a separate doc-polish pass (see #490). Drops the "rolling out in phases" banner pointing at the tracking issue, since the flow it describes is now fully implemented rather than in progress. Also corrects two release-flow descriptions that predated the pypi-release approval gate this phase added: a release no longer lands on PyPI "immediately", it starts a maintainer-gated publish first. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- CONTRIBUTING.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cf31d7a4a..4153b85c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,10 +2,6 @@ Thank you for your interest in contributing. -> **The branching and versioning strategy is rolling out in phases.** See the -> [DevOps tracking issue #490](https://github.com/OvertureMaps/schema/issues/490) -> for current status and what is planned next. - ## Where to send your change This repository uses a two-branch model. Target the branch that matches your @@ -49,8 +45,9 @@ gitGraph main → patch or minor release (version bump) A bug fix or minor feature that bumps the version in the PR and builds the -changelog. On merge, `release-trigger` cuts a published GitHub Release and the -new version lands on PyPI, immediately available to consumers. +changelog. On merge, `release-trigger` cuts a published GitHub Release, which +starts the PyPI publish; a maintainer approves it in the `pypi-release` +environment before it reaches consumers. ```mermaid gitGraph @@ -71,7 +68,8 @@ gitGraph Breaking changes stack on `vnext` until the milestone is ready. Then `vnext` merges into `main` as a regular merge (not a squash), which cuts a published -GitHub Release and puts the new major on PyPI for consumers. +GitHub Release and starts the same maintainer-gated PyPI publish as any other +release. ```mermaid gitGraph From 47c7912af1239a4d2bdecf971f63d3e1a2bb56ac Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 6 Aug 2026 12:28:51 -0400 Subject: [PATCH 07/20] [FIX](ci) Add pull_request trigger to test-schema push never fires for fork PRs (only same-repo branch pushes), so a required check relying solely on push would never run for external contributors, a required status check that can never be satisfied by the PRs GitHub Actions actually needs to gate. Add pull_request with the same path filters; push stays for post-merge validation on main. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/test-schema.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-schema.yaml b/.github/workflows/test-schema.yaml index 5cdb2e3de..802607f41 100644 --- a/.github/workflows/test-schema.yaml +++ b/.github/workflows/test-schema.yaml @@ -9,12 +9,19 @@ on: - 'schema/**' - 'examples/**' - 'counterexamples/**' + pull_request: + paths: + - 'schema/**' + - 'examples/**' + - 'counterexamples/**' permissions: contents: read concurrency: - group: ${{ github.workflow }}-${{ github.ref }} + # push never fires for fork PRs, so pull_request is the only trigger a + # required check can rely on from external contributors. + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: From 46b7e99d197ab0ed8ba99a105c977deefe4b9ace Mon Sep 17 00:00:00 2001 From: John McCall Date: Mon, 10 Aug 2026 13:02:39 -0400 Subject: [PATCH 08/20] [FEATURE](ci) Read overture-releaser PEM from Secrets Manager release-trigger.yaml assumes the narrow gha-releaser-secrets-reader OIDC role and fetches the PEM from omf-github-terraform/releaser/pem instead of reading GHA repo secrets, matching the safe-settings and project-manager app pattern. Client ID is inlined (not sensitive) instead of a secret. Companion Terraform PR wires the role and Secrets Manager entry in omf-github-terraform. Fixes #637. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-publish.yaml | 2 +- .github/workflows/release-trigger.yaml | 29 ++++++++++++++++++++++---- docs/versioning.md | 2 +- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index e3da27146..54d7c2d14 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -1,7 +1,7 @@ name: Release publish # Triggered when a GitHub Release is published (created by release-trigger.yaml -# using the overture-release-publisher app token -- see #637 -- so the native +# using the overture-releaser app token -- see #637 -- so the native # `release: published` event actually fires; GITHUB_TOKEN would not). # # Builds the released package at its released version and publishes it to diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index f15bba743..91d3c12bc 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -16,10 +16,14 @@ name: Publish GitHub release # is a release. No-bump merges publish internal `.postN` builds instead (see # .github/actions/compute-version). # -# Releases are created with the overture-release-publisher app's installation -# token, not GITHUB_TOKEN: GITHUB_TOKEN-created releases don't fire downstream +# Releases are created with the overture-releaser app's installation token, +# not GITHUB_TOKEN: GITHUB_TOKEN-created releases don't fire downstream # `release: published` events, which release-publish.yaml needs to pick up # each release individually. See #637 for provisioning that app. +# +# The app's PEM lives in AWS Secrets Manager (omf-github-terraform), fetched +# here via the narrowly-scoped gha-releaser-secrets-reader OIDC role -- same +# pattern as the safe-settings and project-manager app PEMs. on: push: @@ -63,17 +67,34 @@ jobs: runs-on: ubuntu-slim permissions: contents: read # App token below carries the actual release-creation grant + id-token: write # Required for OIDC authentication to AWS strategy: fail-fast: false # One package's failure must not block sibling releases matrix: include: ${{ fromJSON(needs.detect.outputs.bumps) }} steps: + # Narrow OIDC role (omf-github-terraform's oidc-aws.tf) that can only + # read the releaser app's PEM secret. + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::816069134238:role/gha-releaser-secrets-reader + role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} + + # Exports the PEM as env.RELEASE_PUBLISHER_APP_PEM, masked (incl. multi-line). + - name: Fetch releaser PEM from Secrets Manager + uses: aws-actions/aws-secretsmanager-get-secrets@2cb1a461cbd4865ac4299648312e4704c646cd53 # v3.0.1 + with: + secret-ids: | + RELEASE_PUBLISHER_APP_PEM, omf-github-terraform/releaser/pem + - name: Generate app token id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - client-id: ${{ secrets.RELEASE_PUBLISHER_APP_ID }} # zizmor: ignore[secrets-outside-env] - private-key: ${{ secrets.RELEASE_PUBLISHER_APP_PEM }} # zizmor: ignore[secrets-outside-env] + client-id: Iv23lijru2e660v1zJQO # overture-releaser app ID, not sensitive + private-key: ${{ env.RELEASE_PUBLISHER_APP_PEM }} # zizmor: ignore[secrets-outside-env] # Explicitly scope the app token instead of inheriting all # installation permissions: creating a release/tag is the only # write this workflow performs. diff --git a/docs/versioning.md b/docs/versioning.md index ef78ea187..8721e0997 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -58,7 +58,7 @@ internal builds off the public index by construction. | Version bump merged to `main` | [`release-trigger.yaml`](../.github/workflows/release-trigger.yaml) cuts a GitHub Release per bumped package. | | Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI, gated by the `pypi-release` Environment. | -`release-trigger` creates releases with the `overture-release-publisher` app's +`release-trigger` creates releases with the `overture-releaser` app's installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does not fire the `release: published` event `release-publish` listens for. From 8475a1b841527672fefb9d8b47250392af7b8558 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 10:17:45 -0400 Subject: [PATCH 09/20] fix(actions): treat a package's first version as a releasable bump detect_version_bumps.py no longer skips before:null entries; a brand-new package now flows through release-trigger to PyPI on its first version instead of sitting on CodeArtifact-only .postN builds. detect_affected_packages.py's bumped set is widened to match (any package with a non-null after version) so the same push doesn't also queue an internal build for a package that just got its first release. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect_affected_packages.py | 6 +----- .../detect_version_bumps.py | 17 ++++++++++++----- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/actions/detect-affected-packages/detect_affected_packages.py b/.github/actions/detect-affected-packages/detect_affected_packages.py index 825acec93..4c9e34d7f 100644 --- a/.github/actions/detect-affected-packages/detect_affected_packages.py +++ b/.github/actions/detect-affected-packages/detect_affected_packages.py @@ -34,11 +34,7 @@ def main() -> None: version_diff = json.load(sys.stdin) - bumped = { - change["package"] - for change in version_diff - if change["before"] is not None and change["after"] is not None - } + bumped = {change["package"] for change in version_diff if change["after"] is not None} removed = {change["package"] for change in version_diff if change["after"] is None} changed_paths = json.loads(os.environ.get("CHANGED_DIRS") or "[]") diff --git a/.github/actions/detect-version-bumps/detect_version_bumps.py b/.github/actions/detect-version-bumps/detect_version_bumps.py index fabdd9aef..957e31c92 100644 --- a/.github/actions/detect-version-bumps/detect_version_bumps.py +++ b/.github/actions/detect-version-bumps/detect_version_bumps.py @@ -17,8 +17,9 @@ - Released versions must be plain `..`; PEP 440 variants like `1.2.3rc4` fail loudly. - A version decrease fails: it must never land on main. - - Added packages (`before` null) and removed packages (`after` null) are - not releases; they are skipped. + - Added packages (`before` null) count as a bump: a package's first + version releases to PyPI like any other. Removed packages (`after` + null) are not releases; they are skipped. The `detect-version-bumps` action composes this with `package_versions.py`, which owns reading versions from git (and enforces the major-bump cascade @@ -64,13 +65,19 @@ def main() -> None: before_raw = change["before"] after_raw = change["after"] - if before_raw is None or after_raw is None: - info(f"{package}: added or removed, not a release. Skipping.") + if after_raw is None: + info(f"{package}: removed, not a release. Skipping.") continue - before = semver(package, before_raw) after = semver(package, after_raw) + if before_raw is None: + info(f"{package}: new package at {after_raw} (bump)") + bumps.append({"package": package, "version": after_raw, "tag": f"{package}-v{after_raw}"}) + continue + + before = semver(package, before_raw) + if after < before: errors.append(f"{package}: {before_raw} -> {after_raw}") continue From fff719508b6eddac46786fbeb068c3fb2b14ea6d Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:27:15 -0400 Subject: [PATCH 10/20] [REFACTOR](ci) Un-hoist pyspark expression generation into the package's own prebuild script overture-schema-pyspark ships generated validation expressions that aren't committed to git. main-publish.yaml and release-publish.yaml special-cased it directly (if: ... == 'overture-schema-pyspark' generate step, plus a post-build wheel-unzip check), so both workflows had to know about one package's build quirk. Moves that into packages/overture-schema-pyspark/scripts/prebuild.sh, invoked generically as "run the package's prebuild.sh if it has one". Neither workflow references pyspark by name anymore, and the script's own empty-output guard replaces the wheel-content check. This also keeps the convention backend-agnostic ahead of #623's hatchling -> uv_build migration, which drops hatchling's custom build-hook support entirely. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/main-publish.yaml | 31 +++++++++---------- .../scripts/prebuild.sh | 26 ++++++++++++++++ 2 files changed, 40 insertions(+), 17 deletions(-) create mode 100755 packages/overture-schema-pyspark/scripts/prebuild.sh diff --git a/.github/workflows/main-publish.yaml b/.github/workflows/main-publish.yaml index 8f4779024..33776cbe8 100644 --- a/.github/workflows/main-publish.yaml +++ b/.github/workflows/main-publish.yaml @@ -117,14 +117,20 @@ jobs: context: main index_url: ${{ steps.ca.outputs.index_url }} - # overture-schema-pyspark ships generated validation expressions that are - # not committed to git -- they are regenerated on demand from the - # Pydantic models. `uv build` packages whatever is on disk under the - # module root, so the tree must be generated here before the build, or - # the published wheel ships without its expressions/generated/ modules. - - name: Generate PySpark expressions before build - if: matrix.package == 'overture-schema-pyspark' - run: make generate-pyspark + # overture-schema-pyspark ships generated validation expressions that + # are not committed to git; its packages//scripts/prebuild.sh + # regenerates them before packaging. Every other package has no + # prebuild.sh, so this step is a no-op for them. + - name: Run package's prebuild script, if any + env: + PACKAGE: ${{ matrix.package }} # zizmor: ignore[template-injection] + run: | + set -euo pipefail + script="packages/${PACKAGE}/scripts/prebuild.sh" + if [ -f "$script" ]; then + echo "Running ${script}" + bash "$script" + fi - name: Stamp computed version # Rewrites only the in-memory checkout's pyproject.toml, never @@ -154,15 +160,6 @@ jobs: echo " Wheel file [$wheel] not found. Aborting!" exit 1 fi - # Guard against a silently empty wheel: if the generate step above - # ever regresses, overture-schema-pyspark would publish with no - # validation expressions. Fail loudly instead of shipping a hollow - # package. - if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ - ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then - echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" - exit 1 - fi tarball="dist/${PACKAGE//-/_}-${VERSION}.tar.gz" if [ ! -f "$tarball" ]; then echo " Source tarball file [$tarball] not found. Aborting!" diff --git a/packages/overture-schema-pyspark/scripts/prebuild.sh b/packages/overture-schema-pyspark/scripts/prebuild.sh new file mode 100755 index 000000000..24753ad74 --- /dev/null +++ b/packages/overture-schema-pyspark/scripts/prebuild.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Regenerates the PySpark validation expressions from the Pydantic models. +# The expressions/generated/ tree is not committed to git, and `uv build` +# packages whatever is on disk under the module root, so this must run +# before building or packaging this package or the wheel ships without it. +# +# Invoked as a package-owned convention: CI runs this generically (see +# .github/workflows/main-publish.yaml and release-publish.yaml) as +# `packages//scripts/prebuild.sh`, if the file exists, before +# `uv build --package `. Neither workflow needs to know pyspark is +# special; every other package simply has no prebuild.sh. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +cd "$repo_root" + +output_dir="packages/overture-schema-pyspark/src/overture/schema/pyspark/expressions/generated" +rm -rf "$output_dir" +uv run overture-codegen generate --format pyspark --output-dir "$output_dir" + +# Guard against a silently empty tree: if codegen ever regresses to produce +# nothing, fail loudly here instead of shipping a hollow package to PyPI. +if ! find "$output_dir" -name '*.py' -print -quit | grep -q .; then + echo "::error::No expressions generated under ${output_dir} -- codegen produced nothing." >&2 + exit 1 +fi From a2350e6742733714a3bbca41b101670a1fdefdf9 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:27:56 -0400 Subject: [PATCH 11/20] [FEATURE](ci) Add a workflow_dispatch dry-run to Test PyPI on release-publish Lets a package publish its current on-disk version to Test PyPI on demand, exercising the build-and-publish pipeline without waiting on a real release or touching the production index. Each package gets its own pypi-release- / test-pypi- Environment pair rather than one shared pypi-release/test-pypi pair: a PyPI Trusted Publisher's identity is (repo, workflow filename, environment), so a shared name across packages would let only one package's project name ever bind to it (see #653). Also replaces the post-build wheel-filename check with an upfront `uv version --short` comparison against the release tag: uv has no built-in check for this (astral-sh/uv#9653), and failing before the build runs is cheaper than after. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-publish.yaml | 119 ++++++++++++++++--------- docs/versioning.md | 1 + 2 files changed, 78 insertions(+), 42 deletions(-) diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index 54d7c2d14..255fdb39c 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -5,60 +5,98 @@ name: Release publish # `release: published` event actually fires; GITHUB_TOKEN would not). # # Builds the released package at its released version and publishes it to -# public PyPI via Trusted Publishing (OIDC) + attestations, gated by the -# pypi-release GitHub Environment's required reviewers. +# public PyPI via Trusted Publishing (OIDC) + attestations, gated by that +# package's pypi-release- Environment's required reviewers. # -# Vanity tags (the legacy bare v series continued by overture-schema -# releases; see docs/versioning.md) never get a second GitHub Release object, -# so this only ever fires once per -v release. +# Each package gets its own environment pair rather than one shared +# pypi-release/test-pypi pair: a PyPI Trusted Publisher's identity is (repo, +# workflow filename, environment), so a shared name across packages would let +# only one package's project name ever bind to it (see #653). # # To route approval notifications to Slack instead of the GitHub UI, install # the GitHub app in the target channel and subscribe it to this repository's # deployment reviews -- see # https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-environments-for-deployment # and https://github.com/integrations/slack#subscribing-to-a-repository. -# No workflow changes needed; approval gating stays entirely in the -# pypi-release Environment. +# No workflow changes needed; approval gating stays entirely in each +# pypi-release- Environment. on: release: types: [published] + # Dry-run: publish a package's current on-disk version to Test PyPI + # instead. + workflow_dispatch: + inputs: + package: + description: Package directory name to dry-run publish to Test PyPI (e.g. overture-schema-common) + required: true + type: string permissions: contents: read concurrency: # One release event per package version; never cancel an in-flight publish. - group: ${{ github.workflow }}-${{ github.event.release.tag_name }} + # workflow_dispatch runs key off the chosen package instead of a tag. + group: ${{ github.workflow }}-${{ github.event.release.tag_name || inputs.package }} cancel-in-progress: false jobs: parse: - name: Parse release tag - if: github.event.repository.full_name == github.repository + name: Parse package and version + if: github.event_name == 'workflow_dispatch' || github.event.repository.full_name == github.repository runs-on: ubuntu-slim outputs: package: ${{ steps.parse.outputs.package }} version: ${{ steps.parse.outputs.version }} steps: - - name: Parse package and version from tag + # workflow_dispatch has no release tag to parse; read the package's + # current on-disk version instead, so checkout/uv only run on that path. + - name: Check out repo + if: github.event_name == 'workflow_dispatch' + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Install uv + if: github.event_name == 'workflow_dispatch' + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + enable-cache: false + + - name: Parse package and version id: parse env: + EVENT_NAME: ${{ github.event_name }} TAG: ${{ github.event.release.tag_name }} + INPUT_PACKAGE: ${{ inputs.package }} run: | set -euo pipefail - if [[ ! "$TAG" =~ ^(.+)-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + if [[ ! -d "packages/${INPUT_PACKAGE}" ]]; then + echo "::error::No such package: packages/${INPUT_PACKAGE}" + exit 1 + fi + package="$INPUT_PACKAGE" + version=$(cd "packages/${package}" && uv version --short) + elif [[ "$TAG" =~ ^(.+)-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + package="${BASH_REMATCH[1]}" + version="${BASH_REMATCH[2]}" + else echo "::error::Release tag '${TAG}' is not -v..; nothing to publish." exit 1 fi - echo "package=${BASH_REMATCH[1]}" >> "$GITHUB_OUTPUT" - echo "version=${BASH_REMATCH[2]}" >> "$GITHUB_OUTPUT" + echo "package=${package}" >> "$GITHUB_OUTPUT" + echo "version=${version}" >> "$GITHUB_OUTPUT" publish: - name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to PyPI + name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to ${{ github.event_name == 'workflow_dispatch' && 'Test PyPI' || 'PyPI' }} needs: parse runs-on: ubuntu-latest - environment: pypi-release # Required reviewers gate the PyPI publish step + # Per-package environment name: see #653 (shared name across packages + # would break Trusted Publisher registration). + environment: ${{ github.event_name == 'workflow_dispatch' && format('test-pypi-{0}', needs.parse.outputs.package) || format('pypi-release-{0}', needs.parse.outputs.package) }} permissions: contents: read id-token: write # Required for PyPI Trusted Publishing (OIDC) @@ -75,44 +113,41 @@ jobs: - name: Check out released commit uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ github.event.release.target_commitish }} + ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.release.target_commitish }} persist-credentials: false - name: Sync code to make packages visible to Python run: uv sync --locked --all-packages - # overture-schema-pyspark ships generated validation expressions that are - # not committed to git -- they are regenerated on demand from the - # Pydantic models. `uv build` packages whatever is on disk under the - # module root, so the tree must be generated here before the build, or - # the published wheel ships without its expressions/generated/ modules. - - name: Generate PySpark expressions before build - if: env.PACKAGE == 'overture-schema-pyspark' - run: make generate-pyspark - - - name: Build ${{ env.PACKAGE }} ${{ env.VERSION }} - run: uv build --package "${PACKAGE}" - - - name: Verify built version matches the release + # overture-schema-pyspark's scripts/prebuild.sh regenerates its + # not-committed expressions before packaging; other packages have none. + - name: Run package's prebuild script, if any run: | set -euo pipefail - wheel="dist/${PACKAGE//-/_}-${VERSION}-py3-none-any.whl" - if [ ! -f "$wheel" ]; then - echo " Wheel file [$wheel] not found (built version does not match release ${VERSION}). Aborting!" - exit 1 + script="packages/${PACKAGE}/scripts/prebuild.sh" + if [ -f "$script" ]; then + echo "Running ${script}" + bash "$script" fi - # Guard against a silently empty wheel: if the generate step above - # ever regresses, overture-schema-pyspark would publish with no - # validation expressions. Fail loudly instead of shipping a hollow - # package to public PyPI. - if [ "$PACKAGE" = "overture-schema-pyspark" ] && \ - ! unzip -l "$wheel" | grep -q 'expressions/generated/.*\.py'; then - echo " Wheel [$wheel] has no generated expressions -- codegen did not run. Aborting!" + + # Fails fast if the checked-out commit's version doesn't match the + # release tag. uv has no built-in check for this (astral-sh/uv#9653). + - name: Verify on-disk version matches the release + run: | + set -euo pipefail + on_disk="$(cd "packages/${PACKAGE}" && uv version --short)" + if [ "$on_disk" != "$VERSION" ]; then + echo "::error::packages/${PACKAGE} is at version ${on_disk}, but the release tag says ${VERSION}. Aborting!" exit 1 fi - - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to PyPI + - name: Build ${{ env.PACKAGE }} ${{ env.VERSION }} + run: uv build --package "${PACKAGE}" + + - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to ${{ github.event_name == 'workflow_dispatch' && 'Test PyPI' || 'PyPI' }} uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 with: packages-dir: dist/ attestations: true + repository-url: ${{ github.event_name == 'workflow_dispatch' && 'https://test.pypi.org/legacy/' || '' }} + skip-existing: ${{ github.event_name == 'workflow_dispatch' }} # Don't fail on version conflicts during dry-runs diff --git a/docs/versioning.md b/docs/versioning.md index 944f4a988..7bca8fe5d 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -57,6 +57,7 @@ internal builds off the public index by construction. | Push to `main` | [`main-publish.yaml`](../.github/workflows/main-publish.yaml) detects packages changed without a version bump and publishes their `.postN` build to CodeArtifact. | | Version bump merged to `main` | [`release-trigger.yaml`](../.github/workflows/release-trigger.yaml) cuts a GitHub Release per bumped package. | | Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI, gated by the `pypi-release` Environment. | +| Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package, build it at its current on-disk version, and publish to Test PyPI (`test-pypi` Environment, no approval gate). Exercises the pipeline end to end without a real release or the production index. | `release-trigger` creates releases with the `overture-releaser` app's installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does From 6cd7c163d8dd98d19cb3eb4a57d6911cf40cc6ae Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:37:18 -0400 Subject: [PATCH 12/20] [REFACTOR](ci) Drop the per-environment reviewer gate on PyPI publish 12 packages each needing their own pypi-release- environment (see prior commit, #653) makes per-environment required-reviewer config unruly to maintain. The version-bump PR review is already the approval; nothing further needs to gate the publish once a release exists. Environments stay, scoping only the Trusted Publisher identity (repo, workflow filename, environment) that PyPI's OIDC matching requires. Updates release-publish.yaml's comments, docs/versioning.md, and CONTRIBUTING.md to match, and drops a stale token-scoping comment from release-trigger.yaml that the `permission-contents: write` line already says plainly. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-publish.yaml | 26 ++++++++++---------------- .github/workflows/release-trigger.yaml | 3 --- CONTRIBUTING.md | 7 +++---- docs/versioning.md | 18 ++++++------------ 4 files changed, 19 insertions(+), 35 deletions(-) diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index 255fdb39c..0f6f6b37c 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -5,21 +5,15 @@ name: Release publish # `release: published` event actually fires; GITHUB_TOKEN would not). # # Builds the released package at its released version and publishes it to -# public PyPI via Trusted Publishing (OIDC) + attestations, gated by that -# package's pypi-release- Environment's required reviewers. +# public PyPI via Trusted Publishing (OIDC) + attestations. The version-bump +# PR review is the approval gate; nothing further blocks the publish once a +# release exists. # -# Each package gets its own environment pair rather than one shared -# pypi-release/test-pypi pair: a PyPI Trusted Publisher's identity is (repo, -# workflow filename, environment), so a shared name across packages would let -# only one package's project name ever bind to it (see #653). -# -# To route approval notifications to Slack instead of the GitHub UI, install -# the GitHub app in the target channel and subscribe it to this repository's -# deployment reviews -- see -# https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-environments-for-deployment -# and https://github.com/integrations/slack#subscribing-to-a-repository. -# No workflow changes needed; approval gating stays entirely in each -# pypi-release- Environment. +# Each package gets its own pypi-release-/test-pypi- +# environment pair, used only to scope the Trusted Publisher identity, not +# for approval: a PyPI Trusted Publisher's identity is (repo, workflow +# filename, environment), so a shared name across packages would let only one +# package's project name ever bind to it (see #653). on: release: @@ -94,8 +88,8 @@ jobs: name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to ${{ github.event_name == 'workflow_dispatch' && 'Test PyPI' || 'PyPI' }} needs: parse runs-on: ubuntu-latest - # Per-package environment name: see #653 (shared name across packages - # would break Trusted Publisher registration). + # Per-package environment name, scoping the Trusted Publisher identity + # only (see #653); not an approval gate. environment: ${{ github.event_name == 'workflow_dispatch' && format('test-pypi-{0}', needs.parse.outputs.package) || format('pypi-release-{0}', needs.parse.outputs.package) }} permissions: contents: read diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 0779e80bf..781f17b4e 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -95,9 +95,6 @@ jobs: with: client-id: Iv23lijru2e660v1zJQO # overture-releaser app ID, not sensitive private-key: ${{ env.RELEASE_PUBLISHER_APP_PEM }} # zizmor: ignore[secrets-outside-env] - # Explicitly scope the app token instead of inheriting all - # installation permissions: creating a release/tag is the only - # write this workflow performs. permission-contents: write - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4153b85c2..2cd33b1db 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -46,8 +46,8 @@ gitGraph A bug fix or minor feature that bumps the version in the PR and builds the changelog. On merge, `release-trigger` cuts a published GitHub Release, which -starts the PyPI publish; a maintainer approves it in the `pypi-release` -environment before it reaches consumers. +starts the PyPI publish via Trusted Publishing; the version-bump PR review is +the approval, so nothing further gates it before reaching consumers. ```mermaid gitGraph @@ -68,8 +68,7 @@ gitGraph Breaking changes stack on `vnext` until the milestone is ready. Then `vnext` merges into `main` as a regular merge (not a squash), which cuts a published -GitHub Release and starts the same maintainer-gated PyPI publish as any other -release. +GitHub Release and starts the same PyPI publish as any other release. ```mermaid gitGraph diff --git a/docs/versioning.md b/docs/versioning.md index 7bca8fe5d..e56a209bb 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -56,8 +56,8 @@ internal builds off the public index by construction. |-------|----------| | Push to `main` | [`main-publish.yaml`](../.github/workflows/main-publish.yaml) detects packages changed without a version bump and publishes their `.postN` build to CodeArtifact. | | Version bump merged to `main` | [`release-trigger.yaml`](../.github/workflows/release-trigger.yaml) cuts a GitHub Release per bumped package. | -| Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI, gated by the `pypi-release` Environment. | -| Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package, build it at its current on-disk version, and publish to Test PyPI (`test-pypi` Environment, no approval gate). Exercises the pipeline end to end without a real release or the production index. | +| Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI. | +| Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package, build it at its current on-disk version, and publish to Test PyPI. Exercises the pipeline end to end without a real release or the production index. | `release-trigger` creates releases with the `overture-releaser` app's installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does @@ -183,20 +183,14 @@ changes that package, whether or not it bumps the version. 2. On merge to `main`, `release-trigger` publishes one GitHub Release per bumped package: tag `-v`, notes from that package's `CHANGELOG.md`. -3. Publishing the release starts the PyPI publish. The `pypi-release` - [GitHub Environment](https://docs.github.com/en/actions/how-tos/manage-workflow-runs/manage-environments-for-deployment)'s - required reviewers gate the publish job; approve or reject from the - workflow run's summary page. To get gate notifications in Slack instead of - polling GitHub, subscribe a channel to this repository's deployment - reviews with the - [GitHub app for Slack](https://github.com/integrations/slack#subscribing-to-a-repository) — - no workflow change needed, the gate itself stays entirely in the - Environment's reviewer list. +3. Publishing the release starts the PyPI publish via Trusted Publishing + (OIDC); no further manual approval gates it. The version-bump PR review + is the approval. ```mermaid flowchart LR A[bump + towncrier build
merged to main] --> B[release-trigger:
GitHub Release per package] - B --> C[PyPI publish
maintainer approval] --> D[public PyPI] + B --> C[PyPI publish
Trusted Publishing] --> D[public PyPI] E[no-bump merge] --> F[.postN internal build
CodeArtifact only] ``` From ee9e0f29eb6b9849709be7d1fc1e3cf27b88d3cc Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:40:58 -0400 Subject: [PATCH 13/20] [FIX](ci) Correct release-publisher app name and pin publish checkout to the release tag overture-releaser was never the app's real name; overture-release-publisher is, per #637. Fix it everywhere it drifted in. release-publish's checkout also used target_commitish, which can be a branch name that moves past the release. Use tag_name instead so the publish always builds the exact released commit. Also drops the "explicitly scope the app token" comment above permission-contents: write per feedback -- the field name already says that. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-publish.yaml | 6 ++++-- .github/workflows/release-trigger.yaml | 6 +++--- docs/versioning.md | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index 0f6f6b37c..f7913f06d 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -1,7 +1,7 @@ name: Release publish # Triggered when a GitHub Release is published (created by release-trigger.yaml -# using the overture-releaser app token -- see #637 -- so the native +# using the overture-release-publisher app token -- see #637 -- so the native # `release: published` event actually fires; GITHUB_TOKEN would not). # # Builds the released package at its released version and publishes it to @@ -107,7 +107,9 @@ jobs: - name: Check out released commit uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.release.target_commitish }} + # target_commitish can be a branch name, which may have moved past + # the release; the tag itself pins the exact released commit. + ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.release.tag_name }} persist-credentials: false - name: Sync code to make packages visible to Python diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 781f17b4e..437fe75df 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -16,8 +16,8 @@ name: Publish GitHub release # is a release. No-bump merges publish internal `.postN` builds instead (see # .github/actions/compute-version). # -# Releases are created with the overture-releaser app's installation token, -# not GITHUB_TOKEN: GITHUB_TOKEN-created releases don't fire downstream +# Releases are created with the overture-release-publisher app's installation +# token, not GITHUB_TOKEN: GITHUB_TOKEN-created releases don't fire downstream # `release: published` events, which release-publish.yaml needs to pick up # each release individually. See #637 for provisioning that app. # @@ -93,7 +93,7 @@ jobs: id: app-token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 with: - client-id: Iv23lijru2e660v1zJQO # overture-releaser app ID, not sensitive + client-id: Iv23lijru2e660v1zJQO # overture-release-publisher app ID, not sensitive private-key: ${{ env.RELEASE_PUBLISHER_APP_PEM }} # zizmor: ignore[secrets-outside-env] permission-contents: write diff --git a/docs/versioning.md b/docs/versioning.md index e56a209bb..a6f069f90 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -59,7 +59,7 @@ internal builds off the public index by construction. | Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI. | | Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package, build it at its current on-disk version, and publish to Test PyPI. Exercises the pipeline end to end without a real release or the production index. | -`release-trigger` creates releases with the `overture-releaser` app's +`release-trigger` creates releases with the `overture-release-publisher` app's installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does not fire the `release: published` event `release-publish` listens for. From f8e3b0e2a1b65fd3d7980756a4212638902e55ae Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:42:50 -0400 Subject: [PATCH 14/20] Add changelog fragment for pyspark prebuild un-hoisting Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- packages/overture-schema-pyspark/changelog.d/638.misc.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 packages/overture-schema-pyspark/changelog.d/638.misc.md diff --git a/packages/overture-schema-pyspark/changelog.d/638.misc.md b/packages/overture-schema-pyspark/changelog.d/638.misc.md new file mode 100644 index 000000000..00ac1272e --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/638.misc.md @@ -0,0 +1 @@ +Moved expression-generation from the shared prebuild step into the package's own `scripts/prebuild.sh`. From 2f8a3027bda4f73fd15b1a5b01edb84c9642c952 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:44:00 -0400 Subject: [PATCH 15/20] Expand changelog fragment with CI-only scope note Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- packages/overture-schema-pyspark/changelog.d/638.misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema-pyspark/changelog.d/638.misc.md b/packages/overture-schema-pyspark/changelog.d/638.misc.md index 00ac1272e..1087c5785 100644 --- a/packages/overture-schema-pyspark/changelog.d/638.misc.md +++ b/packages/overture-schema-pyspark/changelog.d/638.misc.md @@ -1 +1 @@ -Moved expression-generation from the shared prebuild step into the package's own `scripts/prebuild.sh`. +Moved expression-generation from the shared prebuild step into the package's own `scripts/prebuild.sh`, run by `release-publish.yaml` before `uv build`. CI-only change: no runtime code, dependencies, or wheel contents affected. From e7fede18a0e9eba80c5bf0626371d0f2c628e817 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 12 Aug 2026 11:49:22 -0400 Subject: [PATCH 16/20] Rename check-python-code's default resolution matrix cell to locked 'default' didn't say what it defaulted to. locked names what actually distinguishes it from lowest-direct: it runs against the committed uv.lock. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/check-python-code.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/check-python-code.yaml b/.github/workflows/check-python-code.yaml index 5f8cb5a47..4fd47b2d7 100644 --- a/.github/workflows/check-python-code.yaml +++ b/.github/workflows/check-python-code.yaml @@ -29,13 +29,13 @@ jobs: strategy: fail-fast: false matrix: - # Default resolution exercises the committed lock against every - # supported Python minor version. The lowest-direct cell pins each - # direct dependency to its declared floor (see UV_RESOLUTION below) - # and runs only on the Python floor, since the resolved-low pyspark - # 3.4 wheels exist for 3.10/3.11 only. + # locked exercises the committed lock against every supported Python + # minor version. The lowest-direct cell pins each direct dependency + # to its declared floor (see UV_RESOLUTION below) and runs only on + # the Python floor, since the resolved-low pyspark 3.4 wheels exist + # for 3.10/3.11 only. python: ["3.10", "3.11", "3.12", "3.13", "3.14"] - resolution: [default] + resolution: [locked] include: - python: "3.10" resolution: lowest-direct @@ -80,11 +80,11 @@ jobs: run: echo "UV_RESOLUTION=lowest-direct" >> "$GITHUB_ENV" # Fail fast if uv.lock is stale (e.g. a pyproject.toml version bump - # landed without a matching `uv lock` run). Only for the default + # landed without a matching `uv lock` run). Only for the locked # resolution cells -- lowest-direct intentionally re-resolves away # from the committed lock, so `--locked` would always fail there. - name: Configure lock check - if: matrix.resolution == 'default' + if: matrix.resolution == 'locked' run: echo "UV_LOCKED=1" >> "$GITHUB_ENV" - name: Run make check From 0c52008755f63889a1d2d8173b894b03a42734e4 Mon Sep 17 00:00:00 2001 From: John McCall Date: Fri, 14 Aug 2026 09:51:33 -0400 Subject: [PATCH 17/20] [FIX](ci) Address review feedback on publish workflows - Downscope main-publish's PR smoke test to the CodeArtifact read-only IAM role instead of the publish role. - Validate the parsed package directory exists for both release-publish arms (release tag and workflow_dispatch), not just workflow_dispatch. - Standardize on the actions/checkout v7.0.1 pin used elsewhere in the repo; main-publish/release-publish had drifted to a mix of v7.0.0/v7.0.1. - Fix wording nit: "in-memory checkout" -> temporary checkout. - detect_affected_packages.py: read version_diff + changed_dirs from a single stdin JSON payload instead of splitting across stdin and an env var, and raise a clear error instead of an IndexError on a malformed changed-directory path. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-affected-packages/action.yml | 5 +- .../detect_affected_packages.py | 50 ++++++++++++------- .github/workflows/main-publish.yaml | 12 +++-- .github/workflows/release-publish.yaml | 25 ++++++---- 4 files changed, 61 insertions(+), 31 deletions(-) diff --git a/.github/actions/detect-affected-packages/action.yml b/.github/actions/detect-affected-packages/action.yml index a99fdade7..3ca5e8399 100644 --- a/.github/actions/detect-affected-packages/action.yml +++ b/.github/actions/detect-affected-packages/action.yml @@ -61,4 +61,7 @@ runs: CHANGED_DIRS: ${{ steps.changed-dirs.outputs.all_modified_files }} DIFF: ${{ steps.diff.outputs.diff }} run: | - echo "$DIFF" | python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" >> "$GITHUB_OUTPUT" + set -euo pipefail + jq -n --argjson version_diff "$DIFF" --argjson changed_dirs "$CHANGED_DIRS" \ + '{version_diff: $version_diff, changed_dirs: $changed_dirs}' \ + | python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-affected-packages/detect_affected_packages.py b/.github/actions/detect-affected-packages/detect_affected_packages.py index 4c9e34d7f..a4da5c62c 100644 --- a/.github/actions/detect-affected-packages/detect_affected_packages.py +++ b/.github/actions/detect-affected-packages/detect_affected_packages.py @@ -3,19 +3,22 @@ """ Detect packages affected by a no-bump push to main. -Composes `package_versions.py diff`'s version-diff JSON (read from stdin) with -a JSON array of changed package directories (env var `CHANGED_DIRS`, from -tj-actions/changed-files' `dir_names` output) to find packages whose directory -changed without also changing their pyproject.toml version. Those need an -internal `.postN` build (see docs/versioning.md). Packages with a version bump -in the same range are excluded because they release via `release-trigger` -instead, and removed packages are excluded because there is nothing left to -build. - -Run from the repository root, piping `package_versions.py diff`'s output in: - - python3 package_versions.py diff BEFORE AFTER \\ - | CHANGED_DIRS='["packages/overture-schema-common"]' python3 detect_affected_packages.py +Composes `package_versions.py diff`'s version-diff JSON with a JSON array of +changed package directories (tj-actions/changed-files' `dir_names` output) to +find packages whose directory changed without also changing their +pyproject.toml version. Those need an internal `.postN` build (see +docs/versioning.md). Packages with a version bump in the same range are +excluded because they release via `release-trigger` instead, and removed +packages are excluded because there is nothing left to build. + +Reads a single JSON object from stdin: {"version_diff": [...], "changed_dirs": +[...]}. Run from the repository root: + + jq -n \\ + --argjson version_diff "$(python3 package_versions.py diff BEFORE AFTER)" \\ + --argjson changed_dirs '["packages/overture-schema-common"]' \\ + '{version_diff: $version_diff, changed_dirs: $changed_dirs}' \\ + | python3 detect_affected_packages.py Prints `$GITHUB_OUTPUT` lines on stdout (progress goes to stderr): @@ -25,20 +28,33 @@ Exit status: 0 Success (including the no-affected case). + 1 Malformed input: a changed_dirs entry isn't packages/. """ import json -import os import sys +def _package_name(path: str) -> str: + """Extract from a packages/ changed-directory entry.""" + parts = path.split("/", 1) + if len(parts) < 2: + raise ValueError( + f"Expected a 'packages/' changed-directory entry, got {path!r}. " + "Check dir_names_max_depth on the changed-files step." + ) + return parts[1] + + def main() -> None: - version_diff = json.load(sys.stdin) + payload = json.load(sys.stdin) + version_diff = payload["version_diff"] + changed_paths = payload["changed_dirs"] + bumped = {change["package"] for change in version_diff if change["after"] is not None} removed = {change["package"] for change in version_diff if change["after"] is None} - changed_paths = json.loads(os.environ.get("CHANGED_DIRS") or "[]") - changed = {path.split("/", 1)[1] for path in changed_paths} + changed = {_package_name(path) for path in changed_paths} affected = sorted(changed - bumped - removed) for package in sorted(changed): diff --git a/.github/workflows/main-publish.yaml b/.github/workflows/main-publish.yaml index 33776cbe8..6e1981a85 100644 --- a/.github/workflows/main-publish.yaml +++ b/.github/workflows/main-publish.yaml @@ -45,7 +45,7 @@ jobs: count: ${{ steps.discover.outputs.count || steps.detect.outputs.count }} packages: ${{ steps.discover.outputs.packages || steps.detect.outputs.packages }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 0 persist-credentials: false @@ -89,18 +89,22 @@ jobs: version: latest - name: Check out code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Sync code to make packages visible to Python run: uv sync --locked --all-packages + # PR smoke test only queries CodeArtifact (via compute-version, below) + # and never publishes -- the publish step is skipped for pull_request. + # Assume the read-only role there so a wiring mistake can't turn the + # smoke test into a real publish. - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 with: aws-region: us-west-2 - role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_Publish + role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_${{ github.event_name == 'pull_request' && 'ReadOnly' || 'Publish' }} role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} - name: Get CodeArtifact credentials @@ -133,7 +137,7 @@ jobs: fi - name: Stamp computed version - # Rewrites only the in-memory checkout's pyproject.toml, never + # Rewrites pyproject.toml only in this job's temporary checkout, never # committed: the released .. stays human-owned, # this just labels the artifact this job builds and publishes. env: diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index f7913f06d..09080d760 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -45,12 +45,14 @@ jobs: package: ${{ steps.parse.outputs.package }} version: ${{ steps.parse.outputs.version }} steps: - # workflow_dispatch has no release tag to parse; read the package's - # current on-disk version instead, so checkout/uv only run on that path. + # Checked out unconditionally (both arms) so the package-existence check + # below applies to a bad release tag too, not just a bad workflow_dispatch + # input. Same ref selection the publish job below uses, so both jobs + # build from the same commit for a given event. - name: Check out repo - if: github.event_name == 'workflow_dispatch' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + ref: ${{ github.event_name == 'workflow_dispatch' && github.sha || github.event.release.tag_name }} persist-credentials: false - name: Install uv @@ -68,12 +70,7 @@ jobs: run: | set -euo pipefail if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then - if [[ ! -d "packages/${INPUT_PACKAGE}" ]]; then - echo "::error::No such package: packages/${INPUT_PACKAGE}" - exit 1 - fi package="$INPUT_PACKAGE" - version=$(cd "packages/${package}" && uv version --short) elif [[ "$TAG" =~ ^(.+)-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then package="${BASH_REMATCH[1]}" version="${BASH_REMATCH[2]}" @@ -81,6 +78,16 @@ jobs: echo "::error::Release tag '${TAG}' is not -v..; nothing to publish." exit 1 fi + + if [[ ! -d "packages/${package}" ]]; then + echo "::error::No such package: packages/${package}" + exit 1 + fi + + if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then + version=$(cd "packages/${package}" && uv version --short) + fi + echo "package=${package}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" @@ -105,7 +112,7 @@ jobs: enable-cache: false # Publish job builds and ships immediately; no cache to poison downstream runs - name: Check out released commit - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: # target_commitish can be a branch name, which may have moved past # the release; the tag itself pins the exact released commit. From c83e62ccd4496e507a02668e269b0b9efcc13bee Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 24 Aug 2026 15:00:32 +0000 Subject: [PATCH 18/20] Fix optional pyspark import guard behavior Co-authored-by: lowlydba <16843041+lowlydba@users.noreply.github.com> --- .../src/overture/schema/pyspark/__init__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py index 7af28fa48..3032dabbb 100644 --- a/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py +++ b/packages/overture-schema-pyspark/src/overture/schema/pyspark/__init__.py @@ -1,7 +1,5 @@ """PySpark validation expressions for Overture Maps data.""" -import importlib.util - from ._pyspark_version import pyspark_version_problem # pyspark is an optional extra (the `spark` extra): a bare install lets this @@ -10,14 +8,16 @@ # import of any submodule -- so a bare install gets an actionable message. A # pyspark that is present but broken still raises its own error, because the # imports below run for real. -if importlib.util.find_spec("pyspark") is None: +try: + import pyspark +except ModuleNotFoundError as exc: + if exc.name != "pyspark": + raise raise ModuleNotFoundError( "overture-schema-pyspark requires PySpark, which isn't installed. " "Install it with `pip install overture-schema-pyspark[spark]`, or run " "in an environment that already provides PySpark (e.g. a Spark cluster)." - ) - -import pyspark + ) from exc # Installing without the extra leaves no resolver to enforce the version floor # declared alongside it, so enforce it here, against the PySpark that actually From 184cf690d501345dc7e05e29a36cf5280fe6a0d9 Mon Sep 17 00:00:00 2001 From: John McCall Date: Mon, 24 Aug 2026 11:38:34 -0400 Subject: [PATCH 19/20] [FEATURE](ci) Add a gated real-PyPI priming path to release-publish's workflow_dispatch PyPI rate-limits pending Trusted Publisher registrations to ~3 at a time, on both pypi.org and test.pypi.org independently (see #653). Priming a publisher past "pending" requires an actual publish, and workflow_dispatch previously only supported Test PyPI, so there was no way to prime real PyPI ahead of a package's first release. workflow_dispatch now takes a target input (test-pypi default, or pypi) and always publishes a synthetic .dev0 instead of the real version: valid PEP 440, ignored by default resolvers, and never collides with the eventual human-owned release. Dispatching to real PyPI uses a new pypi-dispatch- environment with its own required-reviewer gate, since it's the one path that publishes to production PyPI without a version-bump PR behind it; pypi-release- (the automated release path) and test-pypi- are unchanged. This is a bootstrapping tool, not a permanent fixture. Once all 12 packages are primed for the v2.0 launch, the pypi target and its dispatch environments should come out. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-publish.yaml | 67 ++++++++++++++++++++------ docs/versioning.md | 2 +- 2 files changed, 53 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index 09080d760..9bbd7e909 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -9,23 +9,43 @@ name: Release publish # PR review is the approval gate; nothing further blocks the publish once a # release exists. # -# Each package gets its own pypi-release-/test-pypi- -# environment pair, used only to scope the Trusted Publisher identity, not -# for approval: a PyPI Trusted Publisher's identity is (repo, workflow +# Each package gets its own pypi-release-/test-pypi-/ +# pypi-dispatch- environment set, used to scope the Trusted +# Publisher identity: a PyPI Trusted Publisher's identity is (repo, workflow # filename, environment), so a shared name across packages would let only one -# package's project name ever bind to it (see #653). +# package's project name ever bind to it (see #653). pypi-release- +# is not an approval gate, the PR review is. pypi-dispatch- DOES +# carry a required-reviewer gate: it's the one path that publishes to real +# PyPI without a version-bump PR behind it. +# +# workflow_dispatch is a manual priming tool, not a release path: it always +# publishes a synthetic .dev0, never the real version, so it +# can safely convert a package's Trusted Publisher from "pending" to "normal" +# (see docs.pypi.org/trusted-publishers/creating-a-project-through-oidc) +# ahead of that package's actual v2.0 release, on either Test PyPI or real +# PyPI. This exists to work around PyPI's per-account rate limit on pending +# publishers (applies to both pypi.org and test.pypi.org independently, ~3 at +# a time per our testing): register a batch, prime them via dispatch to +# convert them, then register the next batch. Tracked for removal once all +# packages are primed for the v2.0 launch: see #653. on: release: types: [published] - # Dry-run: publish a package's current on-disk version to Test PyPI - # instead. workflow_dispatch: inputs: package: - description: Package directory name to dry-run publish to Test PyPI (e.g. overture-schema-common) + description: Package directory name to prime (e.g. overture-schema-common) required: true type: string + target: + description: Where to publish the priming release + required: false + type: choice + default: test-pypi + options: + - test-pypi + - pypi permissions: contents: read @@ -85,19 +105,25 @@ jobs: fi if [[ "$EVENT_NAME" == "workflow_dispatch" ]]; then - version=$(cd "packages/${package}" && uv version --short) + # .devN is real PEP 440 (unlike the .postN+context.sha local + # versions main-publish.yaml ships, which PyPI/Test PyPI reject + # outright): it never collides with the eventual human-owned + # release version, and default resolvers ignore dev releases, so + # it's safe to actually publish without affecting consumers. Used + # only to prime a package's Trusted Publisher (see #653); never + # the real v2.0 launch artifact. + version="$(cd "packages/${package}" && uv version --short).dev0" fi echo "package=${package}" >> "$GITHUB_OUTPUT" echo "version=${version}" >> "$GITHUB_OUTPUT" publish: - name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to ${{ github.event_name == 'workflow_dispatch' && 'Test PyPI' || 'PyPI' }} + name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to ${{ github.event_name != 'workflow_dispatch' && 'PyPI' || inputs.target == 'pypi' && 'PyPI (priming dispatch)' || 'Test PyPI' }} needs: parse runs-on: ubuntu-latest - # Per-package environment name, scoping the Trusted Publisher identity - # only (see #653); not an approval gate. - environment: ${{ github.event_name == 'workflow_dispatch' && format('test-pypi-{0}', needs.parse.outputs.package) || format('pypi-release-{0}', needs.parse.outputs.package) }} + # See the header comment for what gates which environment. + environment: ${{ github.event_name != 'workflow_dispatch' && format('pypi-release-{0}', needs.parse.outputs.package) || inputs.target == 'pypi' && format('pypi-dispatch-{0}', needs.parse.outputs.package) || format('test-pypi-{0}', needs.parse.outputs.package) }} permissions: contents: read id-token: write # Required for PyPI Trusted Publishing (OIDC) @@ -135,7 +161,11 @@ jobs: # Fails fast if the checked-out commit's version doesn't match the # release tag. uv has no built-in check for this (astral-sh/uv#9653). + # Only meaningful for a real release: workflow_dispatch intentionally + # publishes a synthetic .devN (see the parse job), not the on-disk + # version, so there's nothing to verify here on that path. - name: Verify on-disk version matches the release + if: github.event_name != 'workflow_dispatch' run: | set -euo pipefail on_disk="$(cd "packages/${PACKAGE}" && uv version --short)" @@ -144,13 +174,20 @@ jobs: exit 1 fi + # workflow_dispatch publishes a synthetic .dev0, not the + # on-disk version; stamp it here so the built artifact actually matches + # what gets published. Never committed, this checkout is temporary. + - name: Stamp dispatched version + if: github.event_name == 'workflow_dispatch' + run: uv version "${VERSION}" --package "${PACKAGE}" --frozen + - name: Build ${{ env.PACKAGE }} ${{ env.VERSION }} run: uv build --package "${PACKAGE}" - - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to ${{ github.event_name == 'workflow_dispatch' && 'Test PyPI' || 'PyPI' }} + - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to ${{ github.event_name != 'workflow_dispatch' && 'PyPI' || inputs.target == 'pypi' && 'PyPI (priming dispatch)' || 'Test PyPI' }} uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 with: packages-dir: dist/ attestations: true - repository-url: ${{ github.event_name == 'workflow_dispatch' && 'https://test.pypi.org/legacy/' || '' }} - skip-existing: ${{ github.event_name == 'workflow_dispatch' }} # Don't fail on version conflicts during dry-runs + repository-url: ${{ github.event_name == 'workflow_dispatch' && inputs.target != 'pypi' && 'https://test.pypi.org/legacy/' || '' }} + skip-existing: ${{ github.event_name == 'workflow_dispatch' }} # Idempotent reruns of a priming dispatch; never true for a real release diff --git a/docs/versioning.md b/docs/versioning.md index a6f069f90..b50dd3713 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -57,7 +57,7 @@ internal builds off the public index by construction. | Push to `main` | [`main-publish.yaml`](../.github/workflows/main-publish.yaml) detects packages changed without a version bump and publishes their `.postN` build to CodeArtifact. | | Version bump merged to `main` | [`release-trigger.yaml`](../.github/workflows/release-trigger.yaml) cuts a GitHub Release per bumped package. | | Release published | [`release-publish.yaml`](../.github/workflows/release-publish.yaml) builds that package at its released version and publishes to PyPI. | -| Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package, build it at its current on-disk version, and publish to Test PyPI. Exercises the pipeline end to end without a real release or the production index. | +| Manual dispatch | `release-publish.yaml` also runs on `workflow_dispatch`: pick a package and a target (Test PyPI or real PyPI), build a synthetic `.dev0` (never the on-disk release version), and publish it. Used to prime a package's PyPI Trusted Publisher from "pending" to "normal" ahead of its real release (see [#653](https://github.com/OvertureMaps/schema/issues/653)); real-PyPI dispatches require `pypi-dispatch-` approval since they skip PR review entirely. | `release-trigger` creates releases with the `overture-release-publisher` app's installation token, not `GITHUB_TOKEN`: a `GITHUB_TOKEN`-created release does From 2e7332d7d10211349ef5896dba391552e04eb1b5 Mon Sep 17 00:00:00 2001 From: John McCall Date: Mon, 24 Aug 2026 11:53:15 -0400 Subject: [PATCH 20/20] [REFACTOR](ci) Simplify the priming dispatch to reuse pypi-release-, flag temp blocks for #688 Drops the separate pypi-dispatch- environment: workflow_dispatch already requires repo write access to trigger at all, which is gate enough for a disposable .dev0 priming publish, so a real-PyPI priming dispatch now reuses the same pypi-release- environment the automated release path uses. One environment set per package instead of a third variant. Also marks every pypi-target-specific block with TODO(#688) so the temporary priming path (vs. the permanent test-pypi dispatch) is obvious at the code site, not just in the tracking issue. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-publish.yaml | 38 ++++++++++++++++++-------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-publish.yaml b/.github/workflows/release-publish.yaml index 9bbd7e909..618632ede 100644 --- a/.github/workflows/release-publish.yaml +++ b/.github/workflows/release-publish.yaml @@ -9,14 +9,14 @@ name: Release publish # PR review is the approval gate; nothing further blocks the publish once a # release exists. # -# Each package gets its own pypi-release-/test-pypi-/ -# pypi-dispatch- environment set, used to scope the Trusted -# Publisher identity: a PyPI Trusted Publisher's identity is (repo, workflow -# filename, environment), so a shared name across packages would let only one -# package's project name ever bind to it (see #653). pypi-release- -# is not an approval gate, the PR review is. pypi-dispatch- DOES -# carry a required-reviewer gate: it's the one path that publishes to real -# PyPI without a version-bump PR behind it. +# Each package gets its own pypi-release-/test-pypi- +# environment pair, used to scope the Trusted Publisher identity: a PyPI +# Trusted Publisher's identity is (repo, workflow filename, environment), so +# a shared name across packages would let only one package's project name +# ever bind to it (see #653). Neither environment is an approval gate: the +# version-bump PR review gates real releases, and workflow_dispatch already +# requires repo write access to trigger at all, gate enough for a +# disposable .dev0 priming publish. # # workflow_dispatch is a manual priming tool, not a release path: it always # publishes a synthetic .dev0, never the real version, so it @@ -26,8 +26,14 @@ name: Release publish # PyPI. This exists to work around PyPI's per-account rate limit on pending # publishers (applies to both pypi.org and test.pypi.org independently, ~3 at # a time per our testing): register a batch, prime them via dispatch to -# convert them, then register the next batch. Tracked for removal once all -# packages are primed for the v2.0 launch: see #653. +# convert them, then register the next batch. +# +# TEMPORARY (see #688): the `target: pypi` option below, and everything +# gated on `inputs.target == 'pypi'`, exists only to prime real PyPI's +# Trusted Publishers for the initial v2.0 launch. Tear it out once every +# package has published for real at least once -- `target: test-pypi` (the +# default) is the only permanent path and stays indefinitely for ongoing +# pipeline verification. on: release: @@ -43,6 +49,8 @@ on: required: false type: choice default: test-pypi + # TODO(#688): drop the `pypi` option (and this input entirely, once + # test-pypi is the only choice) after all packages are primed. options: - test-pypi - pypi @@ -119,11 +127,16 @@ jobs: echo "version=${version}" >> "$GITHUB_OUTPUT" publish: + # TODO(#688): the "PyPI (priming dispatch)" branch below goes away with + # the pypi target. name: Publish ${{ needs.parse.outputs.package }} ${{ needs.parse.outputs.version }} to ${{ github.event_name != 'workflow_dispatch' && 'PyPI' || inputs.target == 'pypi' && 'PyPI (priming dispatch)' || 'Test PyPI' }} needs: parse runs-on: ubuntu-latest # See the header comment for what gates which environment. - environment: ${{ github.event_name != 'workflow_dispatch' && format('pypi-release-{0}', needs.parse.outputs.package) || inputs.target == 'pypi' && format('pypi-dispatch-{0}', needs.parse.outputs.package) || format('test-pypi-{0}', needs.parse.outputs.package) }} + # TODO(#688): the `inputs.target == 'pypi' && ...` branch goes away with + # the pypi target -- dispatch will only ever mean Test PyPI at that + # point. + environment: ${{ (github.event_name != 'workflow_dispatch' || inputs.target == 'pypi') && format('pypi-release-{0}', needs.parse.outputs.package) || format('test-pypi-{0}', needs.parse.outputs.package) }} permissions: contents: read id-token: write # Required for PyPI Trusted Publishing (OIDC) @@ -185,6 +198,9 @@ jobs: run: uv build --package "${PACKAGE}" - name: Publish ${{ env.PACKAGE }} ${{ env.VERSION }} to ${{ github.event_name != 'workflow_dispatch' && 'PyPI' || inputs.target == 'pypi' && 'PyPI (priming dispatch)' || 'Test PyPI' }} + # TODO(#688): once the pypi target is gone, this simplifies back to + # `github.event_name == 'workflow_dispatch'` (dispatch always means + # Test PyPI at that point). uses: pypa/gh-action-pypi-publish@ba38be9e461d3875417946c167d0b5f3d385a247 # v1.14.1 with: packages-dir: dist/