From c6399c714c623d792e08c274280d150fb92cab40 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 14 Jul 2026 10:43:31 -0400 Subject: [PATCH 01/41] [FEATURE] Add release-trigger workflow and versioning docs (Phase 2.B) Detects an umbrella overture-schema major/minor bump on main and creates a draft GitHub Release at v..0. Publishing the draft is the only trigger for the public PyPI publish (Phase 3). Decisions locked: patch builds stay CodeArtifact-only; release notes are authored manually at release time; only the umbrella package bump cuts a release. Closes #533 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-trigger.yaml | 110 +++++++++++++++++++++++++ CONTRIBUTING.md | 39 +++++++-- docs/versioning.md | 98 ++++++++++++++++++++++ 3 files changed, 240 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/release-trigger.yaml create mode 100644 docs/versioning.md diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml new file mode 100644 index 000000000..f77c8de3c --- /dev/null +++ b/.github/workflows/release-trigger.yaml @@ -0,0 +1,110 @@ +name: Release trigger + +# Runs on every push to main that touches the umbrella package's pyproject.toml. +# Detects a . version bump of `overture-schema` and cuts a DRAFT +# GitHub Release tagged `v..0`. +# +# A maintainer then writes the release notes and publishes the draft. The +# `release: published` event is the ONLY trigger for a public PyPI publish +# (Phase 3). Patch-level pushes and non-umbrella package bumps never reach +# public PyPI — they publish to CodeArtifact only. +# +# Patch-only changes to the umbrella version are ignored: the patch component +# is computed by CI (see .github/actions/compute-version), not by this +# workflow. + +on: + push: + branches: [main] + paths: + - packages/overture-schema/pyproject.toml + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +jobs: + release-trigger: + name: Detect major/minor bump + if: github.event.repository.full_name == github.repository + runs-on: ubuntu-slim + permissions: + contents: write # Required to create the release and its tag + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Detect major/minor bump + id: detect + env: + BEFORE: ${{ github.event.before }} + AFTER: ${{ github.event.after }} + run: | + set -euo pipefail + + PYPROJECT="packages/overture-schema/pyproject.toml" + + # `before` can be unresolvable after a force-push or history rewrite. + # Treat that as "no bump" rather than failing every subsequent push. + if ! git cat-file -e "${BEFORE}:${PYPROJECT}" 2>/dev/null; then + echo "::warning::Cannot read ${PYPROJECT} at ${BEFORE} — skipping bump detection." + echo "bump=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + + git show "${BEFORE}:${PYPROJECT}" > /tmp/pyproject-before.toml + git show "${AFTER}:${PYPROJECT}" > /tmp/pyproject-after.toml + + python3 - <<'EOF' >> "$GITHUB_OUTPUT" + import sys + import tomllib + + def major_minor(path: str) -> tuple[int, int]: + with open(path, "rb") as f: + version = tomllib.load(f)["project"]["version"] + major, minor, *_ = version.split(".") + return int(major), int(minor) + + before = major_minor("/tmp/pyproject-before.toml") + after = major_minor("/tmp/pyproject-after.toml") + print(f"overture-schema: {before} -> {after}", file=sys.stderr) + + if before == after: + print("bump=false") + else: + print("bump=true") + print(f"tag=v{after[0]}.{after[1]}.0") + EOF + + - name: Create draft release + if: steps.detect.outputs.bump == 'true' + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.detect.outputs.tag }} + TARGET: ${{ github.event.after }} + run: | + set -euo pipefail + + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "::error::Release ${TAG} already exists. A duplicate major/minor bump landed on main — investigate before re-releasing." + exit 1 + fi + + gh release create "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$TARGET" \ + --title "$TAG" \ + --draft \ + --notes "Draft release for ${TAG}. Replace this with the release notes, then publish. Publishing fires the public PyPI release pipeline." + + { + echo "## 📦 Draft release created: ${TAG}" + echo "" + echo "A maintainer must now write release notes and **publish** the draft." + echo "Publishing is the only trigger for the public PyPI release." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dfe6c6e3a..1c46ffc3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -121,6 +121,27 @@ After each push to `main` or `vnext`, CI runs the `compute-versions-dry-run` wor This workflow will be replaced by actual publish workflows in Phase 3. +### Release trigger + +On every push to `main` that touches `packages/overture-schema/pyproject.toml`, the +`release-trigger` workflow compares the umbrella package's `.` before and after. +On a bump it creates a **draft** GitHub Release tagged `v..0`; a maintainer then +writes the release notes and publishes the draft. Publishing the release is the **only** trigger +for a public PyPI publish (Phase 3). Patch-only changes are a no-op. + +See [docs/versioning.md](docs/versioning.md) for the full version scheme, bump guidance, and +release process. + +## Versioning quick reference + +- `.` is a human decision — edit it in `pyproject.toml` in your PR and reset patch + to `0` (e.g. `1.17.1` → `1.18.0`). Minor bumps target `main`; major bumps target `vnext`. +- `` is computed by CI at publish time — never edit it manually. +- Only an `overture-schema` (umbrella) major/minor bump produces a public release. Other + packages publish to CodeArtifact only and reach public PyPI by riding along an umbrella release. +- Details: [docs/versioning.md](docs/versioning.md). + + ## Migration Notes > **Roadmap:** this branching strategy is rolled out in phases, tracked under the parent @@ -132,7 +153,7 @@ This workflow will be replaced by actual publish workflows in Phase 3. | [0](https://github.com/OvertureMaps/schema/issues/506) | ✅ Done | Switch from `dev`/`staging` to the `main`/`vnext` model. | | [1](https://github.com/OvertureMaps/schema/issues/507) | ✅ Done | CI guardrails: PR target check, vnext compatibility check, automatic post-merge rebase. | | [2.A](https://github.com/OvertureMaps/schema/issues/508) | ✅ Done | Version baselines + `compute-version` action. Computes versions only — nothing is published yet. | -| [2.B](https://github.com/OvertureMaps/schema/issues/533) | 🚧 Next | Detect a `.` bump landing on `main` and cut the GitHub Release that triggers a public publish. | +| [2.B](https://github.com/OvertureMaps/schema/issues/533) | ✅ Done | Detect an umbrella `.` bump landing on `main` and cut the draft GitHub Release that gates a public publish. | | [3](https://github.com/OvertureMaps/schema/issues/509) | ⏳ Planned | The actual publish workflows: `vnext` dev builds to CodeArtifact, `main` patch builds, and public PyPI releases. | | [4](https://github.com/OvertureMaps/schema/issues/510) | ⏳ Planned | Documentation polish — diagrams, contributor walkthroughs, FAQ. | @@ -158,17 +179,21 @@ If your fork still references `dev` or `staging`, update your remotes accordingl - `code-artifact` composite action added: replaces the legacy shell script for AWS CodeArtifact auth. - `compute-versions-dry-run` workflow added for version visibility until Phase 3 publish workflows land. -### [Phase 2.B](https://github.com/OvertureMaps/schema/issues/533) +### [Phase 2.B](https://github.com/OvertureMaps/schema/issues/533), July 2026 -Not started. Will add a `p2-release-trigger` workflow that detects a `.` bump -landing on `main` and cuts a GitHub Release — the only trigger for a public PyPI publish. +- `release-trigger` workflow added: detects an umbrella `overture-schema` major/minor bump on + `main` and creates a draft GitHub Release at `v..0`. A maintainer writes notes + and publishes — that `release: published` event is the only public PyPI trigger. +- Decisions locked: patch builds on `main` go to CodeArtifact only; release notes are authored + manually at release time; only the umbrella package's bump cuts a release. +- `docs/versioning.md` added with the full version scheme and release process. ### [Phase 3](https://github.com/OvertureMaps/schema/issues/509) Not started. Will add the actual publish workflows (`p3-dev-builds-ca`, `p3-main-publish`, -`p3-release-publish`) that call the `compute-version` action from Phase 2.A. Where patch -builds on `main` publish to (CodeArtifact-only vs. public PyPI) is still an open decision — -see the linked issue. +`p3-release-publish`) that call the `compute-version` action from Phase 2.A. Patch builds on +`main` publish to CodeArtifact only (decided in Phase 2.B); the public PyPI publish fires on +`release: published`. ### [Phase 4](https://github.com/OvertureMaps/schema/issues/510) diff --git a/docs/versioning.md b/docs/versioning.md new file mode 100644 index 000000000..29fc1fc7c --- /dev/null +++ b/docs/versioning.md @@ -0,0 +1,98 @@ +# Versioning and Releases + +How package versions are computed, when they bump, and how a public release +happens. For branch mechanics and CI guardrails see +[CONTRIBUTING.md](../CONTRIBUTING.md). + +## Version scheme + +All packages under `packages/*` carry a static `..` +version in their `pyproject.toml` (PEP 440). + +| Component | Owner | Meaning | +|-----------|-------|---------| +| `.` | Human | Deliberate, reviewed decision. Edited in `pyproject.toml` via PR. | +| `` | CI | Computed at publish time by the [`compute-version`](../.github/actions/compute-version/action.yml) action. The `pyproject.toml` patch acts only as a floor (baselining). | + +> [!IMPORTANT] +> The umbrella package `overture-schema` is special: only its +> `.` bump triggers a public release. All other packages +> version independently but publish to CodeArtifact only, until they ride +> along an umbrella release. + +## What CI does with the patch component + +| Event | Version formula | Destination | +|-------|-----------------|-------------| +| Push to `vnext` | `+dev.` | CodeArtifact (dev) | +| Push to `main`, no bump | `..` | CodeArtifact only | +| Push to `main` with umbrella major/minor bump | `..0` | Public PyPI (via GitHub Release, see below) | + +> [!NOTE] +> Patch commits to `main` never reach public PyPI. CodeArtifact is the only +> destination for auto-versioned builds. The publish workflows themselves are +> Phase 3 ([#509](https://github.com/OvertureMaps/schema/issues/509)). + +## Bump rules + +### Patch + +Never edit it manually (except baselining). CI computes it. + +### Minor + +Backward-compatible feature or schema addition. Edit `pyproject.toml` in your +PR targeting `main`. + +### Major + +Breaking change. Edit `pyproject.toml` in your PR targeting `vnext`; it +reaches `main` via the release merge below. + +> [!TIP] +> When you bump `` or ``, reset the patch component to `0` +> (e.g. `1.17.1` to `1.18.0`). + +## Release process + +A public PyPI release happens only through this flow: + +1. A PR containing an `overture-schema` `.` bump merges to + `main`. Minor bumps merge directly; major bumps arrive via a + `vnext` to `main` release PR (see below). +2. The [`release-trigger`](../.github/workflows/release-trigger.yaml) + workflow detects the bump and creates a draft GitHub Release tagged + `v..0`. +3. A maintainer writes the release notes on the draft and clicks + "Publish release". Release notes are authored manually at release time; + there is no fragment or auto-generation machinery. +4. The `release: published` event fires the public PyPI publish pipeline + (Phase 3, [#509](https://github.com/OvertureMaps/schema/issues/509)), + which publishes the release's packages. Exact package scope is defined + by the Phase 3 publish workflows. + +```mermaid +flowchart LR + A[major/minor bump
merges to main] --> B[release-trigger:
draft GH Release
v<maj>.<min>.0] + B --> C[maintainer writes notes,
publishes draft] + C --> D[release: published
fires public PyPI publish] + E[patch commit
to main] --> F[CodeArtifact only] +``` + +### vnext to main release PRs + +- Opened by a maintainer when a `vnext` milestone is ready to ship. +- The `vnext compatibility check` and `post-merge vnext rebase` workflows + skip automatically when the PR head is `vnext`. + +> [!WARNING] +> Use a regular merge commit, not squash, so `vnext` history is preserved +> and the post-merge rebase can no-op. + +### Guardrails + +- `release-trigger` fails loudly if the target tag already exists. A + duplicate bump landing on `main` requires human investigation before + re-releasing. +- Non-umbrella package bumps do not create releases. If a non-core package + needs a public release, bump the umbrella package. From 82d0c4234874d4850371d9f0486391327a2b0c80 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 14 Jul 2026 10:52:21 -0400 Subject: [PATCH 02/41] [DOCS] Document umbrella minor bump as escape hatch for non-core releases Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- docs/versioning.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/versioning.md b/docs/versioning.md index 29fc1fc7c..745ec65c3 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -95,4 +95,7 @@ flowchart LR duplicate bump landing on `main` requires human investigation before re-releasing. - Non-umbrella package bumps do not create releases. If a non-core package - needs a public release, bump the umbrella package. + needs a public release now, bump the umbrella package's minor in the same + PR. This is a one-line change and reflects that the umbrella + distribution's contents changed. Otherwise the package reaches public + PyPI with the next umbrella release. From 742ab507b68c80d258e252f3c43efbc7ec91e900 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 14 Jul 2026 10:54:17 -0400 Subject: [PATCH 03/41] [DOCS] Annotate vnext release merge diagram with version bump and draft release tag Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- CONTRIBUTING.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1c46ffc3a..cd2d4252c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -48,11 +48,16 @@ gitGraph commit id: "work B" checkout vnext merge feature-b + commit id: "bump major.minor" checkout main - merge vnext id: "release" + merge vnext id: "release" tag: "v2.0.0 (draft)" commit id: "next work" ``` +The `bump major.minor` commit edits the umbrella package's version in `pyproject.toml`. When the +release merge lands on `main`, CI cuts a draft GitHub Release at that version — see +[docs/versioning.md](docs/versioning.md). + ## Branch Protections Both `main` and `vnext` require a PR and at least two approving reviews before merge. No direct pushes. From 4d1c242583dd5dfdedc770960fa76ca4e78b889a Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 14 Jul 2026 11:00:39 -0400 Subject: [PATCH 04/41] [BUG] Fail release-trigger when umbrella major/minor goes backwards Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-trigger.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index f77c8de3c..d81028a86 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -76,6 +76,14 @@ jobs: if before == after: print("bump=false") + elif after < before: + print( + f"::error::overture-schema major/minor went backwards: " + f"{before[0]}.{before[1]} -> {after[0]}.{after[1]}. " + f"Version decreases must never land on main — revert or fix the version.", + file=sys.stderr, + ) + sys.exit(1) else: print("bump=true") print(f"tag=v{after[0]}.{after[1]}.0") From 2b5e4d2eba4bc180b48ad3cc906f0736754a2a26 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 21 Jul 2026 15:37:47 -0400 Subject: [PATCH 05/41] [FEATURE](devops) Rework Phase 2.B: per-package releases + towncrier changelog - release-trigger: detect per-package major.minor bumps on main and cut a published GitHub Release per package (tag `-v..0`, notes extracted from that package's CHANGELOG.md; umbrella marked Latest) - require-changelog-fragment: new PR check enforcing a changelog update on any major.minor bump (accepts a towncrier fragment or a built CHANGELOG.md) - towncrier: centralize [tool.towncrier] in root pyproject.toml; per-package changelog.d/ with a DRY README that defers to docs/versioning.md (no .gitkeep) - docs/versioning.md: Diataxis rewrite (reference/how-to/why) with a TOC - CONTRIBUTING.md: lean guiding-light rewrite (217 -> 91 lines); richer, workflow-accurate gitGraphs - publish-python-packages: add contents: read for reusable-workflow checkout Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../workflows/publish-python-packages.yaml | 1 + .github/workflows/release-trigger.yaml | 203 +++++++++----- .../workflows/require-changelog-fragment.yaml | 90 +++++++ CONTRIBUTING.md | 249 ++++++------------ docs/versioning.md | 169 ++++++------ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../overture-schema-cli/changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../changelog.d/README.md | 16 ++ .../overture-schema/changelog.d/557.misc.md | 1 + .../overture-schema/changelog.d/README.md | 16 ++ pyproject.toml | 37 +++ 19 files changed, 625 insertions(+), 317 deletions(-) create mode 100644 .github/workflows/require-changelog-fragment.yaml create mode 100644 packages/overture-schema-addresses-theme/changelog.d/README.md create mode 100644 packages/overture-schema-annex/changelog.d/README.md create mode 100644 packages/overture-schema-base-theme/changelog.d/README.md create mode 100644 packages/overture-schema-buildings-theme/changelog.d/README.md create mode 100644 packages/overture-schema-cli/changelog.d/README.md create mode 100644 packages/overture-schema-codegen/changelog.d/README.md create mode 100644 packages/overture-schema-common/changelog.d/README.md create mode 100644 packages/overture-schema-divisions-theme/changelog.d/README.md create mode 100644 packages/overture-schema-places-theme/changelog.d/README.md create mode 100644 packages/overture-schema-system/changelog.d/README.md create mode 100644 packages/overture-schema-transportation-theme/changelog.d/README.md create mode 100644 packages/overture-schema/changelog.d/557.misc.md create mode 100644 packages/overture-schema/changelog.d/README.md diff --git a/.github/workflows/publish-python-packages.yaml b/.github/workflows/publish-python-packages.yaml index c3efbcb04..f76136166 100644 --- a/.github/workflows/publish-python-packages.yaml +++ b/.github/workflows/publish-python-packages.yaml @@ -36,6 +36,7 @@ jobs: if: github.event.repository.full_name == github.repository uses: ./.github/workflows/reusable-check-python-package-versions.yaml permissions: + contents: read # Required for checkout in reusable workflow id-token: write # Required for OIDC in reusable workflow to check AWS CodeArtifact with: before_commit: ${{ github.event.before }} diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index d81028a86..4e0c2c72f 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -1,23 +1,30 @@ name: Release trigger -# Runs on every push to main that touches the umbrella package's pyproject.toml. -# Detects a . version bump of `overture-schema` and cuts a DRAFT -# GitHub Release tagged `v..0`. +# Runs on every push to main that touches any package's pyproject.toml. +# For each package whose . was bumped, cuts a published GitHub +# Release tagged `-v..0`, titled `` `` ``, +# with notes taken from that package's CHANGELOG.md section. # -# A maintainer then writes the release notes and publishes the draft. The -# `release: published` event is the ONLY trigger for a public PyPI publish -# (Phase 3). Patch-level pushes and non-umbrella package bumps never reach -# public PyPI — they publish to CodeArtifact only. +# Packages version independently (see docs/versioning.md). The umbrella +# `overture-schema` release is marked "Latest"; all others are not. # -# Patch-only changes to the umbrella version are ignored: the patch component -# is computed by CI (see .github/actions/compute-version), not by this -# workflow. +# Release notes come from towncrier: a release PR bumps the version and runs +# `uvx towncrier build`, folding that package's changelog.d/ fragments into its +# CHANGELOG.md (reviewed in the PR). This workflow reads back the section. +# +# Patch-only changes are ignored: the patch component is computed by CI (see +# .github/actions/compute-version), not by this workflow. +# +# NOTE: releases are created with GITHUB_TOKEN, which by design does NOT trigger +# further workflow runs. The Phase 3 publish workflow (#509) must therefore be +# triggered by an app/PAT token here or via repository_dispatch, a plain +# `on: release: published` listener will not fire for these releases. on: push: branches: [main] paths: - - packages/overture-schema/pyproject.toml + - packages/*/pyproject.toml permissions: contents: read @@ -28,91 +35,151 @@ concurrency: jobs: release-trigger: - name: Detect major/minor bump + name: Detect version bumps if: github.event.repository.full_name == github.repository runs-on: ubuntu-slim permissions: - contents: write # Required to create the release and its tag + contents: write # Required to create releases and their tags steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: fetch-depth: 0 persist-credentials: false - - name: Detect major/minor bump + - name: Detect version bumps id: detect env: BEFORE: ${{ github.event.before }} - AFTER: ${{ github.event.after }} run: | set -euo pipefail - PYPROJECT="packages/overture-schema/pyproject.toml" - - # `before` can be unresolvable after a force-push or history rewrite. - # Treat that as "no bump" rather than failing every subsequent push. - if ! git cat-file -e "${BEFORE}:${PYPROJECT}" 2>/dev/null; then - echo "::warning::Cannot read ${PYPROJECT} at ${BEFORE} — skipping bump detection." - echo "bump=false" >> "$GITHUB_OUTPUT" - exit 0 - fi - - git show "${BEFORE}:${PYPROJECT}" > /tmp/pyproject-before.toml - git show "${AFTER}:${PYPROJECT}" > /tmp/pyproject-after.toml - - python3 - <<'EOF' >> "$GITHUB_OUTPUT" + python3 - <<'EOF' + import glob + import os + import subprocess import sys import tomllib - def major_minor(path: str) -> tuple[int, int]: - with open(path, "rb") as f: - version = tomllib.load(f)["project"]["version"] + before = os.environ["BEFORE"] + tsv_path = os.path.join(os.environ["RUNNER_TEMP"], "bumps.tsv") + + def major_minor(blob: bytes) -> tuple[int, int]: + version = str(tomllib.loads(blob.decode("utf-8"))["project"]["version"]) major, minor, *_ = version.split(".") return int(major), int(minor) - before = major_minor("/tmp/pyproject-before.toml") - after = major_minor("/tmp/pyproject-after.toml") - print(f"overture-schema: {before} -> {after}", file=sys.stderr) - - if before == after: - print("bump=false") - elif after < before: - print( - f"::error::overture-schema major/minor went backwards: " - f"{before[0]}.{before[1]} -> {after[0]}.{after[1]}. " - f"Version decreases must never land on main — revert or fix the version.", - file=sys.stderr, + bumps: list[tuple[str, str]] = [] + errors: list[str] = [] + + for pyproject in sorted(glob.glob("packages/*/pyproject.toml")): + package = pyproject.split("/")[1] + + # `after` is the working tree, checked out at github.event.after. + with open(pyproject, "rb") as f: + after = major_minor(f.read()) + + # `before` can be unreadable after a force-push, a history rewrite, + # or for a brand-new package. Treat that as "no release" rather + # than failing every subsequent push. + before_blob = subprocess.run( + ["git", "show", f"{before}:{pyproject}"], + capture_output=True, ) + if before_blob.returncode != 0: + print(f"No readable {pyproject} at {before}, skipping {package}.") + continue + + current = major_minor(before_blob.stdout) + if after == current: + continue + if after < current: + errors.append( + f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]}" + ) + continue + + print(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]} (bump)") + bumps.append((package, f"{after[0]}.{after[1]}.0")) + + if errors: + for e in errors: + print( + f"::error::{e}: major/minor went backwards. Version decreases " + "must never land on main; revert or fix the version." + ) sys.exit(1) - else: - print("bump=true") - print(f"tag=v{after[0]}.{after[1]}.0") + + with open(tsv_path, "w", encoding="utf-8") as tsv: + for package, version in bumps: + tsv.write(f"{package}\t{version}\t{package}-v{version}\n") + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"count={len(bumps)}\n") + + if not bumps: + print("No major/minor bumps detected.") EOF - - name: Create draft release - if: steps.detect.outputs.bump == 'true' + - name: Create releases + if: steps.detect.outputs.count != '0' env: GH_TOKEN: ${{ github.token }} - TAG: ${{ steps.detect.outputs.tag }} TARGET: ${{ github.event.after }} + UMBRELLA: overture-schema run: | set -euo pipefail - if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - echo "::error::Release ${TAG} already exists. A duplicate major/minor bump landed on main — investigate before re-releasing." - exit 1 - fi - - gh release create "$TAG" \ - --repo "$GITHUB_REPOSITORY" \ - --target "$TARGET" \ - --title "$TAG" \ - --draft \ - --notes "Draft release for ${TAG}. Replace this with the release notes, then publish. Publishing fires the public PyPI release pipeline." - - { - echo "## 📦 Draft release created: ${TAG}" - echo "" - echo "A maintainer must now write release notes and **publish** the draft." - echo "Publishing is the only trigger for the public PyPI release." - } >> "$GITHUB_STEP_SUMMARY" + cat > "${RUNNER_TEMP}/extract_notes.py" <<'EOF' + import pathlib + import sys + + version, changelog = sys.argv[1], sys.argv[2] + path = pathlib.Path(changelog) + if not path.is_file(): + sys.exit(0) + + lines = path.read_text(encoding="utf-8").splitlines() + start = next( + (i for i, line in enumerate(lines) if line.startswith(f"## [{version}]")), + None, + ) + if start is None: + sys.exit(0) + + end = next( + (j for j in range(start + 1, len(lines)) if lines[j].startswith("## [")), + len(lines), + ) + sys.stdout.write("\n".join(lines[start:end]).strip()) + EOF + + while IFS=$'\t' read -r package version tag; do + [ -z "$package" ] && continue + + if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + echo "::error::Release ${tag} already exists. A duplicate ${package} ${version} bump landed on main; investigate before re-releasing." + exit 1 + fi + + notes=$(python3 "${RUNNER_TEMP}/extract_notes.py" "$version" "packages/${package}/CHANGELOG.md") + if [ -z "$notes" ]; then + notes="Release ${version} of \`${package}\`. No changelog section was found; add towncrier fragments under \`packages/${package}/changelog.d\` and run \`uvx towncrier build --config pyproject.toml --dir packages/${package}\`." + fi + printf '%s\n' "$notes" > "${RUNNER_TEMP}/notes.md" + + latest="--latest=false" + [ "$package" = "$UMBRELLA" ] && latest="--latest" + + gh release create "$tag" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$TARGET" \ + --title "\`${package}\` ${version}" \ + --notes-file "${RUNNER_TEMP}/notes.md" \ + $latest + + { + echo "## 📦 Released \`${package}\` ${version}" + echo "" + echo "Tag \`${tag}\`: published GitHub Release." + } >> "$GITHUB_STEP_SUMMARY" + done < "${RUNNER_TEMP}/bumps.tsv" diff --git a/.github/workflows/require-changelog-fragment.yaml b/.github/workflows/require-changelog-fragment.yaml new file mode 100644 index 000000000..301afb880 --- /dev/null +++ b/.github/workflows/require-changelog-fragment.yaml @@ -0,0 +1,90 @@ +name: Changelog fragment verification + +# Enforces the version/changelog lock-step: any PR that bumps a package's +# . in pyproject.toml must also carry that package's changelog +# update: either a new towncrier fragment under changelog.d/, or a built +# CHANGELOG.md (the result of running `uvx towncrier build`). +# +# Patch-only changes and non-version pyproject edits are exempt: the patch +# component is computed by CI, not by humans (see docs/versioning.md). + +on: + pull_request: + types: [opened, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + check-fragment: + name: Require changelog on version bump + runs-on: ubuntu-latest + permissions: + contents: read # Read pyproject.toml at base/head + pull-requests: read # List PR files + steps: + - name: Require a changelog update for each bumped package + uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const { owner, repo } = context.repo; + const pr = context.payload.pull_request; + + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, repo, pull_number: pr.number, per_page: 100, + }); + + const majorMinor = async (ref, path) => { + try { + const res = await github.rest.repos.getContent({ owner, repo, path, ref }); + const content = Buffer.from(res.data.content, 'base64').toString('utf-8'); + const match = content.match(/^version\s*=\s*["']([^"']+)["']/m); + if (!match) return null; + const [major, minor] = match[1].split('.'); + return `${parseInt(major, 10)}.${parseInt(minor, 10)}`; + } catch (err) { + if (err.status === 404) return null; + throw err; + } + }; + + const fragmentPattern = pkg => + new RegExp(`^packages/${pkg}/changelog\\.d/.+\\.(breaking|feature|bugfix|docs|misc)\\.md$`); + + const bumpedPyprojects = files.filter(f => + /^packages\/[^/]+\/pyproject\.toml$/.test(f.filename)); + + const missing = []; + for (const file of bumpedPyprojects) { + const pkg = file.filename.split('/')[1]; + const before = await majorMinor(pr.base.sha, file.filename); + const after = await majorMinor(pr.head.sha, file.filename); + + // Only a major/minor change on an existing package is a release. + if (!before || !after || before === after) continue; + + const hasChangelog = files.some(f => f.filename === `packages/${pkg}/CHANGELOG.md`); + const hasFragment = files.some(f => + f.status === 'added' && fragmentPattern(pkg).test(f.filename)); + + if (!hasChangelog && !hasFragment) { + missing.push(`\`${pkg}\` (${before} → ${after})`); + } + } + + if (missing.length > 0) { + core.setFailed( + `These packages bump major/minor but include no changelog update:\n` + + missing.map(m => ` - ${m}`).join('\n') + `\n\n` + + `Add a towncrier fragment at packages//changelog.d/..md ` + + `(type: breaking | feature | bugfix | docs | misc), then run ` + + `\`uvx towncrier build --config pyproject.toml --dir packages/\` to fold it into CHANGELOG.md. ` + + `See docs/versioning.md.` + ); + } else { + core.info('Changelog fragment check passed.'); + } diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cd2d4252c..5b3e53061 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,205 +2,106 @@ Thank you for your interest in contributing. -## Branching Strategy +> **The branching and versioning strategy is rolling out in phases.** See the +> [DevOps tracking issue #490](https://github.com/OvertureMaps/schema/issues/490) +> for current status and what is planned next. -> **Work in progress.** This strategy is being rolled out incrementally. See the [DevOps tracking issue #490](https://github.com/OvertureMaps/schema/issues/490) for current status and upcoming phases. +## Where to send your change -This repository uses a two-branch model. Choose your target branch based on the nature of your change. See the [Change Classification](https://lf-overturemaps.atlassian.net/wiki/spaces/SCHEM/pages/14286874/Schema+versioning+and+stability#Change-Classification) wiki page for a detailed breakdown of what constitutes a minor vs. major change. +This repository uses a two-branch model. Target the branch that matches your +change; when in doubt, target `main` and note in your PR if you think it belongs +in `vnext`. The +[Change Classification](https://lf-overturemaps.atlassian.net/wiki/spaces/SCHEM/pages/14286874/Schema+versioning+and+stability#Change-Classification) +wiki page breaks down what counts as a minor vs. major change. -| Branch | Purpose | +| Branch | Use for | |--------|---------| | `main` | Default branch. Bug fixes, minor features, schema improvements. | | `vnext` | Major or breaking changes tied to an active `vnext` milestone. | -When in doubt, target `main` and note in your PR description if you think it belongs in `vnext`. - ### Normal contribution (`main`) +Everyday bug fixes and minor features branch off `main` and merge back through a +PR. Most merges ship as a CI-computed patch; a `major.minor` bump in the PR cuts +a release when it lands. + ```mermaid gitGraph - commit id: "prior work" - commit id: "prior work 2" - branch vnext - branch feature-branch - checkout feature-branch - commit id: "your work" - commit id: "your work 2" + commit id: "overture-schema-v1.17.0" + branch fix-places-brand-enum + checkout fix-places-brand-enum + commit id: "fix brand enum values" + commit id: "add bugfix changelog fragment" checkout main - merge feature-branch id: "merge PR" - commit id: "next work" + merge fix-places-brand-enum id: "PR #561 (patch)" + branch feat-base-land-cover + checkout feat-base-land-cover + commit id: "add land_cover subtype" + commit id: "bump 1.17 to 1.18 + build changelog" + checkout main + merge feat-base-land-cover id: "PR #564 (minor)" tag: "overture-schema-v1.18.0" + commit id: "next fix" ``` ### Major / breaking change (`vnext`) +Breaking changes stack on `vnext` until the milestone is ready. Then `vnext` +merges into `main` as a regular merge (not a squash), which cuts the release. + ```mermaid gitGraph - commit id: "prior work" + commit id: "overture-schema-v1.18.0" branch vnext checkout vnext - branch feature-a - checkout feature-a - commit id: "work A" + branch feat-transportation-access + checkout feat-transportation-access + commit id: "breaking: restructure access" + commit id: "add breaking changelog fragment" checkout vnext - merge feature-a - branch feature-b - checkout feature-b - commit id: "work B" + merge feat-transportation-access id: "PR #570" + branch feat-divisions-hierarchy + checkout feat-divisions-hierarchy + commit id: "breaking: new division hierarchy" + commit id: "add breaking changelog fragment" checkout vnext - merge feature-b - commit id: "bump major.minor" + merge feat-divisions-hierarchy id: "PR #572" + commit id: "bump 1.18 to 2.0 + build changelog" checkout main - merge vnext id: "release" tag: "v2.0.0 (draft)" - commit id: "next work" + merge vnext id: "release merge (not squash)" tag: "overture-schema-v2.0.0" + commit id: "next patch work" ``` -The `bump major.minor` commit edits the umbrella package's version in `pyproject.toml`. When the -release merge lands on `main`, CI cuts a draft GitHub Release at that version — see +The `bump ... + build changelog` commit edits the package version in +`pyproject.toml` and folds its `changelog.d/` fragments into `CHANGELOG.md`. When +the release merge lands on `main`, CI cuts a published GitHub Release tagged +`-v..0` with those notes. See [docs/versioning.md](docs/versioning.md). -## Branch Protections - -Both `main` and `vnext` require a PR and at least two approving reviews before merge. No direct pushes. - -## CI Checks - -### PR target check (advisory) - -Every PR runs an advisory label-vs-target check. It **never blocks** a merge — the reviewer is the -source of truth for change classification. - -| Situation | Warning | -|-----------|---------| -| PR targets `vnext`, label is not `change type - major 🚨` | Consider targeting `main` instead | -| PR targets `main`, label is `change type - major 🚨` | Consider targeting `vnext` instead | - -### vnext compatibility check - -Every PR targeting `main` runs a compatibility check: - -1. The PR is squash-simulated onto `main` in a throwaway clone. -2. `vnext` is dry-run rebased onto the result. -3. If there is no conflict — the check passes silently. -4. If there is a conflict — the check **fails** and CI posts a comment with exact commands. - -**Skipped** for `vnext`→`main` release PRs. - -#### Resolving a vnext conflict - -If this check flags your PR, CI will post a comment listing the conflicting files. Do **not** rebase -your branch onto `vnext` — that would pull unreleased breaking changes into `main`. - -1. See exactly what `vnext` changes in the conflicting files: - ```bash - git fetch origin - git diff origin/main...origin/vnext -- - ``` -2. Open each conflicting file in your editor. The diff above shows what `vnext` adds or changes - there — adjust your edits so they no longer overlap with those lines. -3. Commit the adjustment and push: - ```bash - git add - git commit -m "fix: resolve vnext compatibility" - git push origin your-branch - ``` - -After pushing, the check re-runs automatically. - -### Post-merge vnext rebase - -When any PR merges to `main`, `vnext` is automatically force-rebased onto the new `main` HEAD -using the `overture-pull-requester` GitHub App. - -**Skipped** for `vnext`→`main` release merges — `vnext` is already equal to `main` at that point. - -If the automatic rebase fails, a GitHub issue is opened and assigned to the author of the merged PR. - -> **Accepted tradeoff — in-flight PRs targeting `vnext`:** after the automatic rebase, the base of -> any open PR that targets `vnext` will be force-updated. If you have such a PR open, run -> `git pull --rebase` (or `git fetch origin && git rebase origin/vnext`) on your branch before -> pushing again. - -### Version dry-run (informational) - -After each push to `main` or `vnext`, CI runs the `compute-versions-dry-run` workflow. It logs what package versions **would** be stamped at publish time — no artifacts are actually produced. Check the workflow's job summary for a table of computed versions. - -This workflow will be replaced by actual publish workflows in Phase 3. - -### Release trigger - -On every push to `main` that touches `packages/overture-schema/pyproject.toml`, the -`release-trigger` workflow compares the umbrella package's `.` before and after. -On a bump it creates a **draft** GitHub Release tagged `v..0`; a maintainer then -writes the release notes and publishes the draft. Publishing the release is the **only** trigger -for a public PyPI publish (Phase 3). Patch-only changes are a no-op. - -See [docs/versioning.md](docs/versioning.md) for the full version scheme, bump guidance, and -release process. - -## Versioning quick reference - -- `.` is a human decision — edit it in `pyproject.toml` in your PR and reset patch - to `0` (e.g. `1.17.1` → `1.18.0`). Minor bumps target `main`; major bumps target `vnext`. -- `` is computed by CI at publish time — never edit it manually. -- Only an `overture-schema` (umbrella) major/minor bump produces a public release. Other - packages publish to CodeArtifact only and reach public PyPI by riding along an umbrella release. -- Details: [docs/versioning.md](docs/versioning.md). - - -## Migration Notes - -> **Roadmap:** this branching strategy is rolled out in phases, tracked under the parent -> issue [#490](https://github.com/OvertureMaps/schema/issues/490). When Phases 0-4 are -> complete, this section can be removed in favor of more permanent documentation. - -| Phase | Status | Delivers | -|-------|--------|----------| -| [0](https://github.com/OvertureMaps/schema/issues/506) | ✅ Done | Switch from `dev`/`staging` to the `main`/`vnext` model. | -| [1](https://github.com/OvertureMaps/schema/issues/507) | ✅ Done | CI guardrails: PR target check, vnext compatibility check, automatic post-merge rebase. | -| [2.A](https://github.com/OvertureMaps/schema/issues/508) | ✅ Done | Version baselines + `compute-version` action. Computes versions only — nothing is published yet. | -| [2.B](https://github.com/OvertureMaps/schema/issues/533) | ✅ Done | Detect an umbrella `.` bump landing on `main` and cut the draft GitHub Release that gates a public publish. | -| [3](https://github.com/OvertureMaps/schema/issues/509) | ⏳ Planned | The actual publish workflows: `vnext` dev builds to CodeArtifact, `main` patch builds, and public PyPI releases. | -| [4](https://github.com/OvertureMaps/schema/issues/510) | ⏳ Planned | Documentation polish — diagrams, contributor walkthroughs, FAQ. | - -### [Phase 0](https://github.com/OvertureMaps/schema/issues/506), May 2026 - -- `main` was fast-forwarded to the former `dev` HEAD. -- All open PRs were retargeted `dev` → `main` automatically. -- `dev` and `staging` branches were deleted. -- `vnext` was created from the new `main`. - -If your fork still references `dev` or `staging`, update your remotes accordingly. - -### [Phase 1](https://github.com/OvertureMaps/schema/issues/507), May 2026 - -- Advisory PR target check added: warns when your change-type label and target branch look mismatched. -- vnext compatibility check added: every PR to `main` verifies that `vnext` can rebase cleanly on top; posts exact fix commands on conflict. -- Post-merge automatic rebase added: `vnext` is force-rebased onto `main` after every merge; if it fails, a GitHub issue is opened. - -### [Phase 2.A](https://github.com/OvertureMaps/schema/issues/508), May 2026 - -- All packages baselined with static versions in `pyproject.toml` (`overture-schema` at `1.17.1`, others at `0.1.1`). -- `compute-version` composite action added: computes PEP 440 versions for vnext (dev), main (patch), and main-bump (reset) contexts. -- `code-artifact` composite action added: replaces the legacy shell script for AWS CodeArtifact auth. -- `compute-versions-dry-run` workflow added for version visibility until Phase 3 publish workflows land. - -### [Phase 2.B](https://github.com/OvertureMaps/schema/issues/533), July 2026 - -- `release-trigger` workflow added: detects an umbrella `overture-schema` major/minor bump on - `main` and creates a draft GitHub Release at `v..0`. A maintainer writes notes - and publishes — that `release: published` event is the only public PyPI trigger. -- Decisions locked: patch builds on `main` go to CodeArtifact only; release notes are authored - manually at release time; only the umbrella package's bump cuts a release. -- `docs/versioning.md` added with the full version scheme and release process. - -### [Phase 3](https://github.com/OvertureMaps/schema/issues/509) - -Not started. Will add the actual publish workflows (`p3-dev-builds-ca`, `p3-main-publish`, -`p3-release-publish`) that call the `compute-version` action from Phase 2.A. Patch builds on -`main` publish to CodeArtifact only (decided in Phase 2.B); the public PyPI publish fires on -`release: published`. - -### [Phase 4](https://github.com/OvertureMaps/schema/issues/510) -Not started. Final documentation pass: diagrams, contributor walkthroughs, and an FAQ. No -new procedures — this phase only makes the existing ones easier to read. +## Opening a PR + +- Both `main` and `vnext` require a PR and at least two approving reviews. No + direct pushes. +- An advisory check nudges you if your change-type label and target branch look + mismatched. It never blocks a merge; the reviewer is the source of truth. +- If your change would clash with upcoming `vnext` work, CI fails the PR and + comments the exact commands to fix it. Do not rebase your branch onto `vnext` + yourself; that pulls unreleased changes into `main`. +- If you have an open PR against `vnext`, its base may be force-updated after a + merge to `main`. Run `git pull --rebase` before pushing again. + +## Changing a package version + +- `.` is your call: edit it in `pyproject.toml` and reset patch to + `0` (e.g. `1.17.1` becomes `1.18.0`). Minor bumps target `main`; major bumps + target `vnext`. +- `` is computed by CI at publish time; never edit it manually. +- Every package versions and releases independently. Consumers pin only + `overture-schema`, which pulls in the theme and support packages for a coherent + set. +- A `major.minor` bump **requires a changelog fragment**. Add one under + `packages//changelog.d/` and run + `uvx towncrier build --config pyproject.toml --dir packages/`. CI + enforces it. + +Full version scheme, tag scheme, and release flow: [docs/versioning.md](docs/versioning.md). diff --git a/docs/versioning.md b/docs/versioning.md index 745ec65c3..6d1f34356 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -1,101 +1,120 @@ -# Versioning and Releases +# Versioning and releases -How package versions are computed, when they bump, and how a public release -happens. For branch mechanics and CI guardrails see -[CONTRIBUTING.md](../CONTRIBUTING.md). +Reference and how-to for package versions and releases. Branch mechanics and the +`vnext`/`main` workflow live in [CONTRIBUTING.md](../CONTRIBUTING.md). -## Version scheme +## Contents -All packages under `packages/*` carry a static `..` -version in their `pyproject.toml` (PEP 440). +- [Reference](#reference) + - [Version scheme](#version-scheme) + - [Version → destination](#version--destination) + - [Tag scheme](#tag-scheme) + - [Guardrails](#guardrails) +- [How to](#how-to) + - [Add a changelog fragment](#add-a-changelog-fragment) + - [Cut a release](#cut-a-release) +- [Why](#why) -| Component | Owner | Meaning | -|-----------|-------|---------| -| `.` | Human | Deliberate, reviewed decision. Edited in `pyproject.toml` via PR. | -| `` | CI | Computed at publish time by the [`compute-version`](../.github/actions/compute-version/action.yml) action. The `pyproject.toml` patch acts only as a floor (baselining). | +## Reference -> [!IMPORTANT] -> The umbrella package `overture-schema` is special: only its -> `.` bump triggers a public release. All other packages -> version independently but publish to CodeArtifact only, until they ride -> along an umbrella release. +### Version scheme -## What CI does with the patch component +Every distributable package under `packages/*` carries its own independent +`..` (PEP 440) in its `pyproject.toml`. -| Event | Version formula | Destination | -|-------|-----------------|-------------| -| Push to `vnext` | `+dev.` | CodeArtifact (dev) | -| Push to `main`, no bump | `..` | CodeArtifact only | -| Push to `main` with umbrella major/minor bump | `..0` | Public PyPI (via GitHub Release, see below) | +| Component | Owner | Set by | +|-----------|-------|--------| +| `.` | Human | Edited in `pyproject.toml` via a reviewed PR. | +| `` | CI | [`compute-version`](../.github/actions/compute-version/action.yml) at publish time. The `pyproject.toml` patch is only a floor. | -> [!NOTE] -> Patch commits to `main` never reach public PyPI. CodeArtifact is the only -> destination for auto-versioned builds. The publish workflows themselves are -> Phase 3 ([#509](https://github.com/OvertureMaps/schema/issues/509)). +### Version → destination -## Bump rules +| Event | Version | Destination | +|-------|---------|-------------| +| Push to `vnext` | `+dev.` | CodeArtifact (dev) | +| Push to `main`, no bump | `..` | CodeArtifact | +| `major.minor` bump on `main` | `..0` | Public PyPI | -### Patch +### Tag scheme -Never edit it manually (except baselining). CI computes it. +Each package has its own release series: tag `-v..`, +title `` `` ``. The umbrella `overture-schema` release is +flagged **Latest**. -### Minor +Historical single-series tags (`v0.4.0` … `v1.17.0`) remain valid. The +package-prefixed scheme is new so packages can version independently. This is a +deliberate, one-time discontinuity. -Backward-compatible feature or schema addition. Edit `pyproject.toml` in your -PR targeting `main`. +### Guardrails -### Major +- A changelog update is **required** on any `major.minor` bump, enforced by the + `Changelog fragment verification` check. +- `release-trigger` fails if the target tag already exists, or if a version goes + backwards. -Breaking change. Edit `pyproject.toml` in your PR targeting `vnext`; it -reaches `main` via the release merge below. +## How to -> [!TIP] -> When you bump `` or ``, reset the patch component to `0` -> (e.g. `1.17.1` to `1.18.0`). +### Add a changelog fragment -## Release process +Release notes are assembled from +[towncrier](https://towncrier.readthedocs.io) fragments. Add one per user-facing +change, under the affected package: -A public PyPI release happens only through this flow: +``` +packages//changelog.d/..md +``` -1. A PR containing an `overture-schema` `.` bump merges to - `main`. Minor bumps merge directly; major bumps arrive via a - `vnext` to `main` release PR (see below). -2. The [`release-trigger`](../.github/workflows/release-trigger.yaml) - workflow detects the bump and creates a draft GitHub Release tagged - `v..0`. -3. A maintainer writes the release notes on the draft and clicks - "Publish release". Release notes are authored manually at release time; - there is no fragment or auto-generation machinery. -4. The `release: published` event fires the public PyPI publish pipeline - (Phase 3, [#509](https://github.com/OvertureMaps/schema/issues/509)), - which publishes the release's packages. Exact package scope is defined - by the Phase 3 publish workflows. +| `` | For | +|----------|-----| +| `breaking` | Backward-incompatible changes | +| `feature` | New functionality | +| `bugfix` | Bug fixes | +| `docs` | Documentation-only changes | +| `misc` | Tooling / internal changes | -```mermaid -flowchart LR - A[major/minor bump
merges to main] --> B[release-trigger:
draft GH Release
v<maj>.<min>.0] - B --> C[maintainer writes notes,
publishes draft] - C --> D[release: published
fires public PyPI publish] - E[patch commit
to main] --> F[CodeArtifact only] +The file body is the note itself, written in past tense +(e.g. `Added `provider` to the sources resource.`). Preview the rendered section: + +```bash +# from the repo root +uvx towncrier build --config pyproject.toml --dir packages/ --draft --version ..0 ``` -### vnext to main release PRs +A fragment (or an already-built `CHANGELOG.md` entry) is required on any PR that +bumps that package's `major.minor`. -- Opened by a maintainer when a `vnext` milestone is ready to ship. -- The `vnext compatibility check` and `post-merge vnext rebase` workflows - skip automatically when the PR head is `vnext`. +> [!NOTE] +> The towncrier categories above are defined once in the root `pyproject.toml`. +> A package can override them by adding its own `[tool.towncrier]` block and +> building from that package directory (towncrier replaces, not merges). + +### Cut a release + +1. Bump `.` in the package's `pyproject.toml` (reset patch to `0`), + then run `uvx towncrier build --config pyproject.toml --dir packages/` + from the repo root to fold its fragments into `CHANGELOG.md`. Minor bumps + target `main`; major bumps go via `vnext` and reach `main` through a release + merge. +2. On merge to `main`, `release-trigger` publishes one GitHub Release per bumped + package: tag `-v..0`, notes from that package's + `CHANGELOG.md`. +3. Publishing the release starts the PyPI publish, gated by a maintainer + approval. -> [!WARNING] -> Use a regular merge commit, not squash, so `vnext` history is preserved -> and the post-merge rebase can no-op. +```mermaid +flowchart LR + A[bump + towncrier build
merged to main] --> B[release-trigger:
GitHub Release per package] + B --> C[PyPI publish
maintainer approval] --> D[public PyPI] + E[patch to main] --> F[CodeArtifact only] +``` -### Guardrails +## Why -- `release-trigger` fails loudly if the target tag already exists. A - duplicate bump landing on `main` requires human investigation before - re-releasing. -- Non-umbrella package bumps do not create releases. If a non-core package - needs a public release now, bump the umbrella package's minor in the same - PR. This is a one-line change and reflects that the umbrella - distribution's contents changed. Otherwise the package reaches public - PyPI with the next umbrella release. +- **Human owns `major.minor`, CI owns `patch`.** Release intent is a reviewed + decision; patch numbering is mechanical. +- **Independent per-package versions.** Packages evolve at their own pace. + Consumers pin only `overture-schema`, which depends on the theme/support + packages, giving them a coherent set without tracking each one. +- **towncrier fragments.** Notes are written in context per PR and assembled + automatically, with no merge conflicts on a shared changelog and no + hand-written notes at release time. diff --git a/packages/overture-schema-addresses-theme/changelog.d/README.md b/packages/overture-schema-addresses-theme/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-addresses-theme/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-annex/changelog.d/README.md b/packages/overture-schema-annex/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-annex/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-base-theme/changelog.d/README.md b/packages/overture-schema-base-theme/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-base-theme/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-buildings-theme/changelog.d/README.md b/packages/overture-schema-buildings-theme/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-buildings-theme/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-cli/changelog.d/README.md b/packages/overture-schema-cli/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-cli/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-codegen/changelog.d/README.md b/packages/overture-schema-codegen/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-common/changelog.d/README.md b/packages/overture-schema-common/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-common/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-divisions-theme/changelog.d/README.md b/packages/overture-schema-divisions-theme/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-divisions-theme/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-places-theme/changelog.d/README.md b/packages/overture-schema-places-theme/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-places-theme/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-system/changelog.d/README.md b/packages/overture-schema-system/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-system/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-transportation-theme/changelog.d/README.md b/packages/overture-schema-transportation-theme/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema-transportation-theme/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema/changelog.d/557.misc.md b/packages/overture-schema/changelog.d/557.misc.md new file mode 100644 index 000000000..a75ed3242 --- /dev/null +++ b/packages/overture-schema/changelog.d/557.misc.md @@ -0,0 +1 @@ +Added per-package release-trigger workflow, towncrier changelog fragments, and a fragment-required CI check (Phase 2.B). diff --git a/packages/overture-schema/changelog.d/README.md b/packages/overture-schema/changelog.d/README.md new file mode 100644 index 000000000..c22967575 --- /dev/null +++ b/packages/overture-schema/changelog.d/README.md @@ -0,0 +1,16 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per user-facing change: + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6046d76a6..e8f9e4795 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,3 +76,40 @@ pythonpath = [ "packages/overture-schema/tests", ] verbosity_subtests = 0 + +# Shared changelog (towncrier) config for every package under packages/*. +# Build a package's notes from the repo root: +# uvx towncrier build --config pyproject.toml --dir packages/ --version ..0 +# A package may override these categories by adding its own [tool.towncrier] +# block and building from that package directory (towncrier replaces, not merges). +[tool.towncrier] +directory = "changelog.d" +filename = "CHANGELOG.md" +title_format = "## [{version}] - {project_date}" +issue_format = "[#{issue}](https://github.com/OvertureMaps/schema/issues/{issue})" +ignore = ["README.md"] + +[[tool.towncrier.type]] +directory = "breaking" +name = "Breaking Changes" +showcontent = true + +[[tool.towncrier.type]] +directory = "feature" +name = "Features" +showcontent = true + +[[tool.towncrier.type]] +directory = "bugfix" +name = "Bug Fixes" +showcontent = true + +[[tool.towncrier.type]] +directory = "docs" +name = "Documentation" +showcontent = true + +[[tool.towncrier.type]] +directory = "misc" +name = "Miscellaneous" +showcontent = true \ No newline at end of file From 04158f2da4a16164197218f914559c57092c01b7 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 21 Jul 2026 15:44:10 -0400 Subject: [PATCH 06/41] [DOCS](contributing) Collapsible per-path gitGraphs, slim Opening a PR Wrap each release path (main->patch, main->minor, vnext->major) in a collapsible
section with its own workflow-accurate gitGraph. Drop the branch-protection bullet and reduce the CI notes to authoritative links to the vnext-compat and pr-advisory workflows, which comment their own fix steps inline. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- CONTRIBUTING.md | 67 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5b3e53061..19a52b52a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,11 +19,13 @@ wiki page breaks down what counts as a minor vs. major change. | `main` | Default branch. Bug fixes, minor features, schema improvements. | | `vnext` | Major or breaking changes tied to an active `vnext` milestone. | -### Normal contribution (`main`) +Three common paths take a branch to a merge. Expand each for the commit-level flow. -Everyday bug fixes and minor features branch off `main` and merge back through a -PR. Most merges ship as a CI-computed patch; a `major.minor` bump in the PR cuts -a release when it lands. +
+main → patch (no version bump) + +Everyday bug fixes and schema tweaks. You do not touch the version; CI computes +the patch at publish time and no GitHub Release is cut. ```mermaid gitGraph @@ -31,19 +33,36 @@ gitGraph branch fix-places-brand-enum checkout fix-places-brand-enum commit id: "fix brand enum values" - commit id: "add bugfix changelog fragment" + commit id: "add bugfix fragment" checkout main - merge fix-places-brand-enum id: "PR #561 (patch)" + merge fix-places-brand-enum id: "PR #561" + commit id: "more fixes" +``` + +
+ +
+main → minor release (version bump) + +A minor feature that bumps `major.minor` in the PR and builds the changelog. On +merge, `release-trigger` cuts the release. + +```mermaid +gitGraph + commit id: "overture-schema-v1.17.0" branch feat-base-land-cover checkout feat-base-land-cover commit id: "add land_cover subtype" commit id: "bump 1.17 to 1.18 + build changelog" checkout main - merge feat-base-land-cover id: "PR #564 (minor)" tag: "overture-schema-v1.18.0" - commit id: "next fix" + merge feat-base-land-cover id: "PR #564" tag: "overture-schema-v1.18.0" + commit id: "next work" ``` -### Major / breaking change (`vnext`) +
+ +
+vnext → major release Breaking changes stack on `vnext` until the milestone is ready. Then `vnext` merges into `main` as a regular merge (not a squash), which cuts the release. @@ -56,13 +75,13 @@ gitGraph branch feat-transportation-access checkout feat-transportation-access commit id: "breaking: restructure access" - commit id: "add breaking changelog fragment" + commit id: "add breaking fragment" checkout vnext merge feat-transportation-access id: "PR #570" branch feat-divisions-hierarchy checkout feat-divisions-hierarchy commit id: "breaking: new division hierarchy" - commit id: "add breaking changelog fragment" + commit id: "add breaking fragment" checkout vnext merge feat-divisions-hierarchy id: "PR #572" commit id: "bump 1.18 to 2.0 + build changelog" @@ -71,24 +90,26 @@ gitGraph commit id: "next patch work" ``` +
+ The `bump ... + build changelog` commit edits the package version in -`pyproject.toml` and folds its `changelog.d/` fragments into `CHANGELOG.md`. When -the release merge lands on `main`, CI cuts a published GitHub Release tagged +`pyproject.toml` and folds its `changelog.d/` fragments into `CHANGELOG.md`. On +merge to `main`, CI cuts a published GitHub Release tagged `-v..0` with those notes. See [docs/versioning.md](docs/versioning.md). - ## Opening a PR -- Both `main` and `vnext` require a PR and at least two approving reviews. No - direct pushes. -- An advisory check nudges you if your change-type label and target branch look - mismatched. It never blocks a merge; the reviewer is the source of truth. -- If your change would clash with upcoming `vnext` work, CI fails the PR and - comments the exact commands to fix it. Do not rebase your branch onto `vnext` - yourself; that pulls unreleased changes into `main`. -- If you have an open PR against `vnext`, its base may be force-updated after a - merge to `main`. Run `git pull --rebase` before pushing again. +A couple of CI checks comment on your PR when they need something. Each explains +itself inline, so follow the comment it leaves rather than a copy here: + +- [vnext compatibility check](.github/workflows/vnext-compat.yaml): fails and + posts the fix if your change clashes with unreleased `vnext` work. +- [PR advisory check](.github/workflows/pr-advisory.yaml): nudges you on a likely + change-type / target-branch mismatch. Advisory only; the reviewer decides. + +If you have an open PR against `vnext`, its base may be force-updated after a +merge to `main`; run `git pull --rebase` before pushing again. ## Changing a package version From 396174df7a9da96f7b70bd33ecf06979ad19e029 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 21 Jul 2026 15:48:00 -0400 Subject: [PATCH 07/41] [DOCS](contributing) Note release/availability per path; tidy Opening a PR Add how/when each release path reaches consumers (patch rides the next release to PyPI via the umbrella; minor and major publish to PyPI on merge). Reword the Opening a PR CI notes to point at the checks without narrating. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- CONTRIBUTING.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 19a52b52a..5414bf312 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,7 +25,8 @@ Three common paths take a branch to a merge. Expand each for the commit-level fl main → patch (no version bump) Everyday bug fixes and schema tweaks. You do not touch the version; CI computes -the patch at publish time and no GitHub Release is cut. +the patch at publish time and no GitHub Release is cut. The fix ships in the next +release someone cuts (the next minor or major), rolled into that version. ```mermaid gitGraph @@ -45,7 +46,8 @@ gitGraph main → minor release (version bump) A minor feature that bumps `major.minor` in the PR and builds the changelog. On -merge, `release-trigger` cuts the release. +merge, `release-trigger` cuts a published GitHub Release and the new version lands +on PyPI, immediately available to consumers. ```mermaid gitGraph @@ -65,7 +67,8 @@ gitGraph vnext → major release Breaking changes stack on `vnext` until the milestone is ready. Then `vnext` -merges into `main` as a regular merge (not a squash), which cuts the release. +merges into `main` as a regular merge (not a squash), which cuts a published +GitHub Release and puts the new major on PyPI for consumers. ```mermaid gitGraph @@ -100,14 +103,15 @@ merge to `main`, CI cuts a published GitHub Release tagged ## Opening a PR -A couple of CI checks comment on your PR when they need something. Each explains -itself inline, so follow the comment it leaves rather than a copy here: +Two CI checks may comment on your PR: - [vnext compatibility check](.github/workflows/vnext-compat.yaml): fails and posts the fix if your change clashes with unreleased `vnext` work. -- [PR advisory check](.github/workflows/pr-advisory.yaml): nudges you on a likely +- [PR advisory check](.github/workflows/pr-advisory.yaml): flags a likely change-type / target-branch mismatch. Advisory only; the reviewer decides. +Follow the comment each check leaves on the PR. + If you have an open PR against `vnext`, its base may be force-updated after a merge to `main`; run `git pull --rebase` before pushing again. From 77a7b25042dd25e688c47050299bd1743c70c211 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 21 Jul 2026 16:01:51 -0400 Subject: [PATCH 08/41] [REFACTOR](ci) Extract release-trigger Python into script files Move the inline detect heredoc and the extract_notes heredoc out of release-trigger.yaml into .github/workflows/scripts/detect_version_bumps.py and extract_release_notes.py, matching the existing package-versions.py convention. The workflow steps now just invoke them, so the Python is lint-covered, testable, and free of heredoc quoting. Behavior is unchanged. Also vary the CONTRIBUTING release diagrams across distinct packages (places/base/transportation themes) and note that a no-bump merge still publishes a patch to internal CodeArtifact. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-trigger.yaml | 96 +--------------- .../workflows/scripts/detect_version_bumps.py | 106 ++++++++++++++++++ .../scripts/extract_release_notes.py | 49 ++++++++ CONTRIBUTING.md | 37 +++--- 4 files changed, 176 insertions(+), 112 deletions(-) create mode 100644 .github/workflows/scripts/detect_version_bumps.py create mode 100644 .github/workflows/scripts/extract_release_notes.py diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 4e0c2c72f..c54d563e0 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -50,75 +50,7 @@ jobs: id: detect env: BEFORE: ${{ github.event.before }} - run: | - set -euo pipefail - - python3 - <<'EOF' - import glob - import os - import subprocess - import sys - import tomllib - - before = os.environ["BEFORE"] - tsv_path = os.path.join(os.environ["RUNNER_TEMP"], "bumps.tsv") - - def major_minor(blob: bytes) -> tuple[int, int]: - version = str(tomllib.loads(blob.decode("utf-8"))["project"]["version"]) - major, minor, *_ = version.split(".") - return int(major), int(minor) - - bumps: list[tuple[str, str]] = [] - errors: list[str] = [] - - for pyproject in sorted(glob.glob("packages/*/pyproject.toml")): - package = pyproject.split("/")[1] - - # `after` is the working tree, checked out at github.event.after. - with open(pyproject, "rb") as f: - after = major_minor(f.read()) - - # `before` can be unreadable after a force-push, a history rewrite, - # or for a brand-new package. Treat that as "no release" rather - # than failing every subsequent push. - before_blob = subprocess.run( - ["git", "show", f"{before}:{pyproject}"], - capture_output=True, - ) - if before_blob.returncode != 0: - print(f"No readable {pyproject} at {before}, skipping {package}.") - continue - - current = major_minor(before_blob.stdout) - if after == current: - continue - if after < current: - errors.append( - f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]}" - ) - continue - - print(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]} (bump)") - bumps.append((package, f"{after[0]}.{after[1]}.0")) - - if errors: - for e in errors: - print( - f"::error::{e}: major/minor went backwards. Version decreases " - "must never land on main; revert or fix the version." - ) - sys.exit(1) - - with open(tsv_path, "w", encoding="utf-8") as tsv: - for package, version in bumps: - tsv.write(f"{package}\t{version}\t{package}-v{version}\n") - - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: - out.write(f"count={len(bumps)}\n") - - if not bumps: - print("No major/minor bumps detected.") - EOF + run: python3 ./.github/workflows/scripts/detect_version_bumps.py - name: Create releases if: steps.detect.outputs.count != '0' @@ -129,30 +61,6 @@ jobs: run: | set -euo pipefail - cat > "${RUNNER_TEMP}/extract_notes.py" <<'EOF' - import pathlib - import sys - - version, changelog = sys.argv[1], sys.argv[2] - path = pathlib.Path(changelog) - if not path.is_file(): - sys.exit(0) - - lines = path.read_text(encoding="utf-8").splitlines() - start = next( - (i for i, line in enumerate(lines) if line.startswith(f"## [{version}]")), - None, - ) - if start is None: - sys.exit(0) - - end = next( - (j for j in range(start + 1, len(lines)) if lines[j].startswith("## [")), - len(lines), - ) - sys.stdout.write("\n".join(lines[start:end]).strip()) - EOF - while IFS=$'\t' read -r package version tag; do [ -z "$package" ] && continue @@ -161,7 +69,7 @@ jobs: exit 1 fi - notes=$(python3 "${RUNNER_TEMP}/extract_notes.py" "$version" "packages/${package}/CHANGELOG.md") + notes=$(python3 ./.github/workflows/scripts/extract_release_notes.py "$version" "packages/${package}/CHANGELOG.md") if [ -z "$notes" ]; then notes="Release ${version} of \`${package}\`. No changelog section was found; add towncrier fragments under \`packages/${package}/changelog.d\` and run \`uvx towncrier build --config pyproject.toml --dir packages/${package}\`." fi diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/workflows/scripts/detect_version_bumps.py new file mode 100644 index 000000000..7560e6444 --- /dev/null +++ b/.github/workflows/scripts/detect_version_bumps.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 + +""" +Detect per-package . version bumps between two commits on main. + +Run from the repository root by the `Release trigger` workflow. Compares each +`packages/*/pyproject.toml` at the pushed commit (the working tree) against its +content at the `before` commit, and records the packages whose . +increased. Patch-only changes are ignored: the patch component is computed by +CI, not by humans (see docs/versioning.md). + +Environment: + BEFORE The `before` commit SHA (github.event.before). + RUNNER_TEMP Directory for the emitted bumps.tsv. + GITHUB_OUTPUT Step output file; receives `count=`. + +Outputs: + $RUNNER_TEMP/bumps.tsv One `\t\t` row per bump. + count= Number of bumps, written to $GITHUB_OUTPUT. + +Exit status: + 0 Success (including the no-bump case). + 1 A package's . went backwards; that must never land on main. +""" + +from pathlib import Path +import glob +import os +import subprocess +import sys +import tomllib + + +def major_minor(blob: bytes) -> tuple[int, int]: + """Parse `project.version` from pyproject.toml bytes into (major, minor).""" + version = str(tomllib.loads(blob.decode("utf-8"))["project"]["version"]) + major, minor, *_ = version.split(".") + return int(major), int(minor) + + +def before_major_minor(before: str, pyproject: str) -> tuple[int, int] | None: + """ + Return the (major, minor) of `pyproject` at the `before` commit. + + The `before` blob can be unreadable after a force-push, a history rewrite, + or for a brand-new package. Treat that as "no previous version" rather than + failing every subsequent push. + """ + result = subprocess.run( + ["git", "show", f"{before}:{pyproject}"], + capture_output=True, + ) + if result.returncode != 0: + return None + return major_minor(result.stdout) + + +def main() -> None: + before = os.environ["BEFORE"] + tsv_path = os.path.join(os.environ["RUNNER_TEMP"], "bumps.tsv") + + bumps: list[tuple[str, str]] = [] + errors: list[str] = [] + + for pyproject in sorted(glob.glob("packages/*/pyproject.toml")): + package = pyproject.split("/")[1] + + with open(pyproject, "rb") as f: + after = major_minor(f.read()) + + current = before_major_minor(before, pyproject) + if current is None: + print(f"No readable {pyproject} at {before}, skipping {package}.") + continue + + if after == current: + continue + + if after < current: + errors.append(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]}") + continue + + print(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]} (bump)") + bumps.append((package, f"{after[0]}.{after[1]}.0")) + + if errors: + for e in errors: + print( + f"::error::{e}: major/minor went backwards. Version decreases " + "must never land on main; revert or fix the version." + ) + sys.exit(1) + + with open(tsv_path, "w", encoding="utf-8") as tsv: + for package, version in bumps: + tsv.write(f"{package}\t{version}\t{package}-v{version}\n") + + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: + out.write(f"count={len(bumps)}\n") + + if not bumps: + print("No major/minor bumps detected.") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/scripts/extract_release_notes.py b/.github/workflows/scripts/extract_release_notes.py new file mode 100644 index 000000000..90932483b --- /dev/null +++ b/.github/workflows/scripts/extract_release_notes.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +""" +Extract one package's release notes from its CHANGELOG.md. + +Run by the `Release trigger` workflow. Prints the changelog section for the +given version (the block from `## []` up to the next `## [` heading, +stripped). Prints nothing if the changelog or the section is absent, so the +caller can fall back to a default message. + +Usage: + extract_release_notes.py +""" + +from pathlib import Path +import sys + + +def extract(version: str, changelog: str) -> str: + """Return the trimmed `## []` section, or "" if not found.""" + path = Path(changelog) + if not path.is_file(): + return "" + + lines = path.read_text(encoding="utf-8").splitlines() + start = next( + (i for i, line in enumerate(lines) if line.startswith(f"## [{version}]")), + None, + ) + if start is None: + return "" + + end = next( + (j for j in range(start + 1, len(lines)) if lines[j].startswith("## [")), + len(lines), + ) + return "\n".join(lines[start:end]).strip() + + +def main() -> None: + if len(sys.argv) != 3: + print(f"Usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(2) + + sys.stdout.write(extract(sys.argv[1], sys.argv[2])) + + +if __name__ == "__main__": + main() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5414bf312..d5b7ca2f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,19 +24,20 @@ Three common paths take a branch to a merge. Expand each for the commit-level fl
main → patch (no version bump) -Everyday bug fixes and schema tweaks. You do not touch the version; CI computes -the patch at publish time and no GitHub Release is cut. The fix ships in the next -release someone cuts (the next minor or major), rolled into that version. +Everyday bug fixes and schema tweaks. You do not touch the version; on merge CI +publishes the next patch (`..`) to internal +CodeArtifact. No GitHub Release or public PyPI release is cut; the fix reaches +public PyPI with the next minor or major that ships. ```mermaid gitGraph - commit id: "overture-schema-v1.17.0" + commit id: "places-theme-v0.4.0" branch fix-places-brand-enum checkout fix-places-brand-enum commit id: "fix brand enum values" commit id: "add bugfix fragment" checkout main - merge fix-places-brand-enum id: "PR #561" + merge fix-places-brand-enum id: "PR #561 (CodeArtifact 0.4.1)" commit id: "more fixes" ``` @@ -51,13 +52,13 @@ on PyPI, immediately available to consumers. ```mermaid gitGraph - commit id: "overture-schema-v1.17.0" + commit id: "base-theme-v0.1.0" branch feat-base-land-cover checkout feat-base-land-cover commit id: "add land_cover subtype" - commit id: "bump 1.17 to 1.18 + build changelog" + commit id: "bump 0.1 to 0.2 + build changelog" checkout main - merge feat-base-land-cover id: "PR #564" tag: "overture-schema-v1.18.0" + merge feat-base-land-cover id: "PR #564" tag: "base-theme-v0.2.0" commit id: "next work" ``` @@ -72,24 +73,24 @@ GitHub Release and puts the new major on PyPI for consumers. ```mermaid gitGraph - commit id: "overture-schema-v1.18.0" + commit id: "transportation-theme-v0.5.0" branch vnext checkout vnext - branch feat-transportation-access - checkout feat-transportation-access + branch feat-access-restructure + checkout feat-access-restructure commit id: "breaking: restructure access" commit id: "add breaking fragment" checkout vnext - merge feat-transportation-access id: "PR #570" - branch feat-divisions-hierarchy - checkout feat-divisions-hierarchy - commit id: "breaking: new division hierarchy" + merge feat-access-restructure id: "PR #570" + branch feat-segment-model + checkout feat-segment-model + commit id: "breaking: new segment model" commit id: "add breaking fragment" checkout vnext - merge feat-divisions-hierarchy id: "PR #572" - commit id: "bump 1.18 to 2.0 + build changelog" + merge feat-segment-model id: "PR #572" + commit id: "bump 0.5 to 1.0 + build changelog" checkout main - merge vnext id: "release merge (not squash)" tag: "overture-schema-v2.0.0" + merge vnext id: "release merge (not squash)" tag: "transportation-theme-v1.0.0" commit id: "next patch work" ``` From 4e1044701af3dcf66a29acf4028808baf102c464 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 21 Jul 2026 16:22:35 -0400 Subject: [PATCH 09/41] [REFACTOR](ci) Add create-package-release action; require fragment on any package change Extract the release-creation logic into a reusable composite action, .github/actions/create-package-release, with a typed input contract and a dry-run mode. release-trigger now runs as two jobs: detect emits the bumped packages as a JSON step output, and a fail-fast:false matrix job calls the action once per package, so one bad CHANGELOG cannot block sibling releases. extract_release_notes.py moves into the action as an internal detail, and detect_version_bumps.py emits JSON (and uses pathlib for portability). Broaden the changelog policy: require-changelog-fragment now flags any package with changed files (excluding its own changelog.d/ and CHANGELOG.md) that lacks a fragment, not just major/minor bumps. Update docs/versioning.md and CONTRIBUTING.md to match, and clarify that a no-bump merge still publishes a patch to internal CodeArtifact that is consumable immediately. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../actions/create-package-release/action.yml | 115 ++++++++++++++++++ .../extract_release_notes.py | 0 .github/workflows/release-trigger.yaml | 69 +++++------ .../workflows/require-changelog-fragment.yaml | 66 +++++----- .../workflows/scripts/detect_version_bumps.py | 31 +++-- CONTRIBUTING.md | 7 +- docs/versioning.md | 11 +- 7 files changed, 196 insertions(+), 103 deletions(-) create mode 100644 .github/actions/create-package-release/action.yml rename .github/{workflows/scripts => actions/create-package-release}/extract_release_notes.py (100%) diff --git a/.github/actions/create-package-release/action.yml b/.github/actions/create-package-release/action.yml new file mode 100644 index 000000000..96ba10965 --- /dev/null +++ b/.github/actions/create-package-release/action.yml @@ -0,0 +1,115 @@ +name: Create package release +description: > + Creates a published GitHub Release for a single package from its towncrier + CHANGELOG.md. + + Reads the `## []` section of `packages//CHANGELOG.md` for + the release notes (falling back to a pointer message if absent), fails if the + target tag already exists, and creates the release tagged + `-v` at `target`. + + Prerequisites: repo must be checked out and the GitHub CLI (`gh`) available + (both true on GitHub-hosted runners). + +inputs: + package: + description: Package directory name under packages/ (e.g. overture-schema). + required: true + version: + description: Release version, major.minor.0 (e.g. 1.18.0). + required: true + tag: + description: Release tag (e.g. overture-schema-v1.18.0). + required: true + target: + description: Commit SHA the tag should point at. + required: true + latest: + description: Whether to mark this release as "Latest" (true/false). + required: false + default: "false" + dry-run: + description: > + When "true", resolve and print what would be released without creating + anything. Read-only: a pre-existing tag is reported as a warning, not a + failure. + required: false + default: "false" + token: + description: "Token used to create the release (needs contents: write)." + required: true + +outputs: + release-url: + description: URL of the created release, or empty on a dry run. + value: ${{ steps.release.outputs.release-url }} + +runs: + using: composite + steps: + - name: Create release + id: release + shell: bash + env: + GH_TOKEN: ${{ inputs.token }} + PACKAGE: ${{ inputs.package }} + VERSION: ${{ inputs.version }} + TAG: ${{ inputs.tag }} + TARGET: ${{ inputs.target }} + LATEST: ${{ inputs.latest }} + DRY_RUN: ${{ inputs.dry-run }} + run: | + set -euo pipefail + + tag_exists=false + if gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then + tag_exists=true + fi + + if [ "$tag_exists" = "true" ] && [ "$DRY_RUN" != "true" ]; then + echo "::error::Release ${TAG} already exists. A duplicate ${PACKAGE} ${VERSION} bump landed on main; investigate before re-releasing." + exit 1 + fi + + changelog="packages/${PACKAGE}/CHANGELOG.md" + notes=$(python3 "${GITHUB_ACTION_PATH}/extract_release_notes.py" "$VERSION" "$changelog") + if [ -z "$notes" ]; then + notes="Release ${VERSION} of \`${PACKAGE}\`. No changelog section was found; add towncrier fragments under \`packages/${PACKAGE}/changelog.d\` and run \`uvx towncrier build --config pyproject.toml --dir packages/${PACKAGE}\`." + fi + printf '%s\n' "$notes" > "${RUNNER_TEMP}/notes.md" + + latest_flag="--latest=false" + [ "$LATEST" = "true" ] && latest_flag="--latest" + + if [ "$DRY_RUN" = "true" ]; then + [ "$tag_exists" = "true" ] && echo "::warning::Release ${TAG} already exists; a real run would fail here." + { + echo "## 🔍 Dry run: \`${PACKAGE}\` ${VERSION}" + echo "" + echo "Would create tag \`${TAG}\` at \`${TARGET}\` (${latest_flag})." + echo "" + echo "
Notes" + echo "" + cat "${RUNNER_TEMP}/notes.md" + echo "" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" + echo "release-url=" >> "$GITHUB_OUTPUT" + exit 0 + fi + + gh release create "$TAG" \ + --repo "$GITHUB_REPOSITORY" \ + --target "$TARGET" \ + --title "\`${PACKAGE}\` ${VERSION}" \ + --notes-file "${RUNNER_TEMP}/notes.md" \ + $latest_flag + + url=$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json url --jq .url) + echo "release-url=${url}" >> "$GITHUB_OUTPUT" + + { + echo "## 📦 Released \`${PACKAGE}\` ${VERSION}" + echo "" + echo "Tag \`${TAG}\`: [published GitHub Release](${url})." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/scripts/extract_release_notes.py b/.github/actions/create-package-release/extract_release_notes.py similarity index 100% rename from .github/workflows/scripts/extract_release_notes.py rename to .github/actions/create-package-release/extract_release_notes.py diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index c54d563e0..1ef112aae 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -34,12 +34,15 @@ concurrency: cancel-in-progress: false jobs: - release-trigger: + detect: name: Detect version bumps if: github.event.repository.full_name == github.repository runs-on: ubuntu-slim permissions: - contents: write # Required to create releases and their tags + contents: read # Read pyproject.toml history to detect bumps + outputs: + count: ${{ steps.detect.outputs.count }} + bumps: ${{ steps.detect.outputs.bumps }} steps: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: @@ -52,42 +55,28 @@ jobs: BEFORE: ${{ github.event.before }} run: python3 ./.github/workflows/scripts/detect_version_bumps.py - - name: Create releases - if: steps.detect.outputs.count != '0' - env: - GH_TOKEN: ${{ github.token }} - TARGET: ${{ github.event.after }} - UMBRELLA: overture-schema - run: | - set -euo pipefail - - while IFS=$'\t' read -r package version tag; do - [ -z "$package" ] && continue - - if gh release view "$tag" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1; then - echo "::error::Release ${tag} already exists. A duplicate ${package} ${version} bump landed on main; investigate before re-releasing." - exit 1 - fi - - notes=$(python3 ./.github/workflows/scripts/extract_release_notes.py "$version" "packages/${package}/CHANGELOG.md") - if [ -z "$notes" ]; then - notes="Release ${version} of \`${package}\`. No changelog section was found; add towncrier fragments under \`packages/${package}/changelog.d\` and run \`uvx towncrier build --config pyproject.toml --dir packages/${package}\`." - fi - printf '%s\n' "$notes" > "${RUNNER_TEMP}/notes.md" - - latest="--latest=false" - [ "$package" = "$UMBRELLA" ] && latest="--latest" - - gh release create "$tag" \ - --repo "$GITHUB_REPOSITORY" \ - --target "$TARGET" \ - --title "\`${package}\` ${version}" \ - --notes-file "${RUNNER_TEMP}/notes.md" \ - $latest + release: + name: Release ${{ matrix.package }} ${{ matrix.version }} + needs: detect + if: needs.detect.outputs.count != '0' + runs-on: ubuntu-slim + permissions: + contents: write # Required to create releases and their tags + strategy: + fail-fast: false # One package's failure must not block sibling releases + matrix: + include: ${{ fromJSON(needs.detect.outputs.bumps) }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false - { - echo "## 📦 Released \`${package}\` ${version}" - echo "" - echo "Tag \`${tag}\`: published GitHub Release." - } >> "$GITHUB_STEP_SUMMARY" - done < "${RUNNER_TEMP}/bumps.tsv" + - name: Create release + uses: ./.github/actions/create-package-release + with: + package: ${{ matrix.package }} + version: ${{ matrix.version }} + tag: ${{ matrix.tag }} + target: ${{ github.event.after }} + latest: ${{ matrix.package == 'overture-schema' }} + token: ${{ github.token }} diff --git a/.github/workflows/require-changelog-fragment.yaml b/.github/workflows/require-changelog-fragment.yaml index 301afb880..d14f0a180 100644 --- a/.github/workflows/require-changelog-fragment.yaml +++ b/.github/workflows/require-changelog-fragment.yaml @@ -1,12 +1,14 @@ name: Changelog fragment verification -# Enforces the version/changelog lock-step: any PR that bumps a package's -# . in pyproject.toml must also carry that package's changelog -# update: either a new towncrier fragment under changelog.d/, or a built -# CHANGELOG.md (the result of running `uvx towncrier build`). +# Enforces the changelog policy: any PR that changes a package must also carry +# that package's changelog update, either a new towncrier fragment under +# changelog.d/, or a built CHANGELOG.md (the result of running +# `uvx towncrier build`). # -# Patch-only changes and non-version pyproject edits are exempt: the patch -# component is computed by CI, not by humans (see docs/versioning.md). +# "Changes a package" means any file under packages// other than that +# package's own changelog artifacts (changelog.d/ and CHANGELOG.md). Every +# user-facing change gets a fragment, regardless of whether it bumps the +# version (see docs/versioning.md). on: pull_request: @@ -21,13 +23,13 @@ concurrency: jobs: check-fragment: - name: Require changelog on version bump + name: Require changelog on package change runs-on: ubuntu-latest permissions: - contents: read # Read pyproject.toml at base/head + contents: read # Read PR file list pull-requests: read # List PR files steps: - - name: Require a changelog update for each bumped package + - name: Require a changelog update for each changed package uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 with: script: | @@ -38,48 +40,36 @@ jobs: owner, repo, pull_number: pr.number, per_page: 100, }); - const majorMinor = async (ref, path) => { - try { - const res = await github.rest.repos.getContent({ owner, repo, path, ref }); - const content = Buffer.from(res.data.content, 'base64').toString('utf-8'); - const match = content.match(/^version\s*=\s*["']([^"']+)["']/m); - if (!match) return null; - const [major, minor] = match[1].split('.'); - return `${parseInt(major, 10)}.${parseInt(minor, 10)}`; - } catch (err) { - if (err.status === 404) return null; - throw err; - } - }; - const fragmentPattern = pkg => new RegExp(`^packages/${pkg}/changelog\\.d/.+\\.(breaking|feature|bugfix|docs|misc)\\.md$`); - const bumpedPyprojects = files.filter(f => - /^packages\/[^/]+\/pyproject\.toml$/.test(f.filename)); - - const missing = []; - for (const file of bumpedPyprojects) { - const pkg = file.filename.split('/')[1]; - const before = await majorMinor(pr.base.sha, file.filename); - const after = await majorMinor(pr.head.sha, file.filename); + // A package "changed" if any of its files changed other than its own + // changelog artifacts (the fragment or the built CHANGELOG.md). + const isChangelogArtifact = (pkg, path) => + path.startsWith(`packages/${pkg}/changelog.d/`) || + path === `packages/${pkg}/CHANGELOG.md`; - // Only a major/minor change on an existing package is a release. - if (!before || !after || before === after) continue; + const touched = new Set(); + for (const file of files) { + const match = file.filename.match(/^packages\/([^/]+)\//); + if (!match) continue; + const pkg = match[1]; + if (!isChangelogArtifact(pkg, file.filename)) touched.add(pkg); + } + const missing = []; + for (const pkg of [...touched].sort()) { const hasChangelog = files.some(f => f.filename === `packages/${pkg}/CHANGELOG.md`); const hasFragment = files.some(f => f.status === 'added' && fragmentPattern(pkg).test(f.filename)); - if (!hasChangelog && !hasFragment) { - missing.push(`\`${pkg}\` (${before} → ${after})`); - } + if (!hasChangelog && !hasFragment) missing.push(pkg); } if (missing.length > 0) { core.setFailed( - `These packages bump major/minor but include no changelog update:\n` + - missing.map(m => ` - ${m}`).join('\n') + `\n\n` + + `These packages changed but include no changelog update:\n` + + missing.map(m => ` - \`${m}\``).join('\n') + `\n\n` + `Add a towncrier fragment at packages//changelog.d/..md ` + `(type: breaking | feature | bugfix | docs | misc), then run ` + `\`uvx towncrier build --config pyproject.toml --dir packages/\` to fold it into CHANGELOG.md. ` + diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/workflows/scripts/detect_version_bumps.py index 7560e6444..7696a60e2 100644 --- a/.github/workflows/scripts/detect_version_bumps.py +++ b/.github/workflows/scripts/detect_version_bumps.py @@ -11,12 +11,12 @@ Environment: BEFORE The `before` commit SHA (github.event.before). - RUNNER_TEMP Directory for the emitted bumps.tsv. - GITHUB_OUTPUT Step output file; receives `count=`. + GITHUB_OUTPUT Step output file; receives `count` and `bumps`. -Outputs: - $RUNNER_TEMP/bumps.tsv One `\t\t` row per bump. - count= Number of bumps, written to $GITHUB_OUTPUT. +Outputs (written to $GITHUB_OUTPUT): + count= Number of bumped packages. + bumps= JSON array of {"package", "version", "tag"} objects, one per + bump. Consumed as a matrix by the release job. Exit status: 0 Success (including the no-bump case). @@ -24,7 +24,7 @@ """ from pathlib import Path -import glob +import json import os import subprocess import sys @@ -57,16 +57,15 @@ def before_major_minor(before: str, pyproject: str) -> tuple[int, int] | None: def main() -> None: before = os.environ["BEFORE"] - tsv_path = os.path.join(os.environ["RUNNER_TEMP"], "bumps.tsv") - bumps: list[tuple[str, str]] = [] + bumps: list[dict[str, str]] = [] errors: list[str] = [] - for pyproject in sorted(glob.glob("packages/*/pyproject.toml")): - package = pyproject.split("/")[1] + for path in sorted(Path("packages").glob("*/pyproject.toml")): + package = path.parent.name + pyproject = path.as_posix() # git wants forward slashes on every OS - with open(pyproject, "rb") as f: - after = major_minor(f.read()) + after = major_minor(path.read_bytes()) current = before_major_minor(before, pyproject) if current is None: @@ -80,8 +79,9 @@ def main() -> None: errors.append(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]}") continue + version = f"{after[0]}.{after[1]}.0" print(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]} (bump)") - bumps.append((package, f"{after[0]}.{after[1]}.0")) + bumps.append({"package": package, "version": version, "tag": f"{package}-v{version}"}) if errors: for e in errors: @@ -91,12 +91,9 @@ def main() -> None: ) sys.exit(1) - with open(tsv_path, "w", encoding="utf-8") as tsv: - for package, version in bumps: - tsv.write(f"{package}\t{version}\t{package}-v{version}\n") - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: out.write(f"count={len(bumps)}\n") + out.write(f"bumps={json.dumps(bumps)}\n") if not bumps: print("No major/minor bumps detected.") diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d5b7ca2f4..affb5f2c3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,8 +26,9 @@ Three common paths take a branch to a merge. Expand each for the commit-level fl Everyday bug fixes and schema tweaks. You do not touch the version; on merge CI publishes the next patch (`..`) to internal -CodeArtifact. No GitHub Release or public PyPI release is cut; the fix reaches -public PyPI with the next minor or major that ships. +CodeArtifact, where it is consumable immediately. No GitHub Release is cut and +nothing new lands on public PyPI; the patch reaches public PyPI only when the +next minor or major release ships. ```mermaid gitGraph @@ -125,7 +126,7 @@ merge to `main`; run `git pull --rebase` before pushing again. - Every package versions and releases independently. Consumers pin only `overture-schema`, which pulls in the theme and support packages for a coherent set. -- A `major.minor` bump **requires a changelog fragment**. Add one under +- Any change to a package **requires a changelog fragment**. Add one under `packages//changelog.d/` and run `uvx towncrier build --config pyproject.toml --dir packages/`. CI enforces it. diff --git a/docs/versioning.md b/docs/versioning.md index 6d1f34356..81a197f72 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -47,8 +47,8 @@ deliberate, one-time discontinuity. ### Guardrails -- A changelog update is **required** on any `major.minor` bump, enforced by the - `Changelog fragment verification` check. +- A changelog fragment is **required** on any change to a package, enforced by + the `Changelog fragment verification` check. - `release-trigger` fails if the target tag already exists, or if a version goes backwards. @@ -57,8 +57,9 @@ deliberate, one-time discontinuity. ### Add a changelog fragment Release notes are assembled from -[towncrier](https://towncrier.readthedocs.io) fragments. Add one per user-facing -change, under the affected package: +[towncrier](https://towncrier.readthedocs.io) fragments. Add one for every change +to a package, including patch-level fixes and internal work (use the `misc` +type), under the affected package: ``` packages//changelog.d/..md @@ -81,7 +82,7 @@ uvx towncrier build --config pyproject.toml --dir packages/ --draft --v ``` A fragment (or an already-built `CHANGELOG.md` entry) is required on any PR that -bumps that package's `major.minor`. +changes that package, whether or not it bumps the version. > [!NOTE] > The towncrier categories above are defined once in the root `pyproject.toml`. From 69bbe1daf6b99d42de3a80556cce7b0e615c520f Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 21 Jul 2026 17:01:49 -0400 Subject: [PATCH 10/41] [DOCS](changelog) Add pyspark changelog keeper; align keeper wording with any-change policy The new overture-schema-pyspark package (merged from main) lacked a changelog.d/ keeper. Add one so the fragment check and towncrier treat it like the other packages. Also correct the stale 'per user-facing change' line in all keepers to match the any-change policy in docs/versioning.md. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../changelog.d/README.md | 3 ++- .../overture-schema-annex/changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- .../overture-schema-cli/changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- .../changelog.d/README.md | 17 +++++++++++++++++ .../changelog.d/README.md | 3 ++- .../changelog.d/README.md | 3 ++- packages/overture-schema/changelog.d/README.md | 3 ++- 13 files changed, 41 insertions(+), 12 deletions(-) create mode 100644 packages/overture-schema-pyspark/changelog.d/README.md diff --git a/packages/overture-schema-addresses-theme/changelog.d/README.md b/packages/overture-schema-addresses-theme/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-addresses-theme/changelog.d/README.md +++ b/packages/overture-schema-addresses-theme/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-annex/changelog.d/README.md b/packages/overture-schema-annex/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-annex/changelog.d/README.md +++ b/packages/overture-schema-annex/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-base-theme/changelog.d/README.md b/packages/overture-schema-base-theme/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-base-theme/changelog.d/README.md +++ b/packages/overture-schema-base-theme/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-buildings-theme/changelog.d/README.md b/packages/overture-schema-buildings-theme/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-buildings-theme/changelog.d/README.md +++ b/packages/overture-schema-buildings-theme/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-cli/changelog.d/README.md b/packages/overture-schema-cli/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-cli/changelog.d/README.md +++ b/packages/overture-schema-cli/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-codegen/changelog.d/README.md b/packages/overture-schema-codegen/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-codegen/changelog.d/README.md +++ b/packages/overture-schema-codegen/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-common/changelog.d/README.md b/packages/overture-schema-common/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-common/changelog.d/README.md +++ b/packages/overture-schema-common/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-divisions-theme/changelog.d/README.md b/packages/overture-schema-divisions-theme/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-divisions-theme/changelog.d/README.md +++ b/packages/overture-schema-divisions-theme/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-places-theme/changelog.d/README.md b/packages/overture-schema-places-theme/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-places-theme/changelog.d/README.md +++ b/packages/overture-schema-places-theme/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-pyspark/changelog.d/README.md b/packages/overture-schema-pyspark/changelog.d/README.md new file mode 100644 index 000000000..93297ce0d --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/README.md @@ -0,0 +1,17 @@ +# Changelog fragments + +[towncrier](https://towncrier.readthedocs.io) news fragments for this package. +One file per change to this package (including patch-level fixes and internal +work): + +``` +changelog.d/..md +``` + +Types, body format, the preview command, and when a fragment is required are +documented once in +[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). + +> [!NOTE] +> This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is +> needed. Leave it in place even when the directory holds no fragments. \ No newline at end of file diff --git a/packages/overture-schema-system/changelog.d/README.md b/packages/overture-schema-system/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-system/changelog.d/README.md +++ b/packages/overture-schema-system/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema-transportation-theme/changelog.d/README.md b/packages/overture-schema-transportation-theme/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema-transportation-theme/changelog.d/README.md +++ b/packages/overture-schema-transportation-theme/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md diff --git a/packages/overture-schema/changelog.d/README.md b/packages/overture-schema/changelog.d/README.md index c22967575..93297ce0d 100644 --- a/packages/overture-schema/changelog.d/README.md +++ b/packages/overture-schema/changelog.d/README.md @@ -1,7 +1,8 @@ # Changelog fragments [towncrier](https://towncrier.readthedocs.io) news fragments for this package. -One file per user-facing change: +One file per change to this package (including patch-level fixes and internal +work): ``` changelog.d/..md From ceb4d3d3911a2afa1ea55ca9842c42e8c98eb890 Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 23 Jul 2026 09:17:15 -0400 Subject: [PATCH 11/41] Update docs/versioning.md Co-authored-by: Seth Fitzsimmons Signed-off-by: John McCall --- docs/versioning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioning.md b/docs/versioning.md index 81a197f72..fcafdaddf 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -20,7 +20,7 @@ Reference and how-to for package versions and releases. Branch mechanics and the ### Version scheme Every distributable package under `packages/*` carries its own independent -`..` (PEP 440) in its `pyproject.toml`. +`..` ([PEP 440](https://peps.python.org/pep-0440/)) in its `pyproject.toml`. | Component | Owner | Set by | |-----------|-------|--------| From da328e88eb508b3f1d46bdfa895d644f24d180e0 Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 23 Jul 2026 09:28:37 -0400 Subject: [PATCH 12/41] Update docs/versioning.md Co-authored-by: Seth Fitzsimmons Signed-off-by: John McCall --- docs/versioning.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/versioning.md b/docs/versioning.md index fcafdaddf..a32bc4071 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -61,7 +61,7 @@ Release notes are assembled from to a package, including patch-level fixes and internal work (use the `misc` type), under the affected package: -``` +```text packages//changelog.d/..md ``` From 3565b1b6465c3558590d61f5aa1ddfcdb93b193b Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 23 Jul 2026 09:29:52 -0400 Subject: [PATCH 13/41] Update packages/overture-schema/changelog.d/557.misc.md Signed-off-by: John McCall --- packages/overture-schema/changelog.d/557.misc.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema/changelog.d/557.misc.md b/packages/overture-schema/changelog.d/557.misc.md index a75ed3242..79ecd1dc4 100644 --- a/packages/overture-schema/changelog.d/557.misc.md +++ b/packages/overture-schema/changelog.d/557.misc.md @@ -1 +1 @@ -Added per-package release-trigger workflow, towncrier changelog fragments, and a fragment-required CI check (Phase 2.B). +Added per-package release-trigger workflow, towncrier changelog fragments, and a fragment-required CI check. From f14f5b18494da9a320c90c210355a8d2d6d0e70e Mon Sep 17 00:00:00 2001 From: John McCall Date: Thu, 23 Jul 2026 16:22:24 -0400 Subject: [PATCH 14/41] Update packages/overture-schema-addresses-theme/changelog.d/README.md Co-authored-by: Seth Fitzsimmons Signed-off-by: John McCall --- packages/overture-schema-addresses-theme/changelog.d/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/overture-schema-addresses-theme/changelog.d/README.md b/packages/overture-schema-addresses-theme/changelog.d/README.md index 93297ce0d..1cdb08bb4 100644 --- a/packages/overture-schema-addresses-theme/changelog.d/README.md +++ b/packages/overture-schema-addresses-theme/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` From 341feda0843af2dfa3a8eb20a93a96de89cb70f8 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 15:01:53 -0400 Subject: [PATCH 15/41] [FEATURE](devops) Human-owned patch releases + PEP 440 post internal builds Rework per the John/Vic versioning session: - detect_version_bumps.py compares the full major.minor.patch triple, so a deliberate patch bump now cuts a release; also resolves hatch dynamic versions (fixes a crash on overture-schema-pyspark). - compute-version drops the CodeArtifact query entirely; internal builds are .post+main. / +vnext., which order after the released version and stay off public PyPI by construction. - compute-versions-dry-run no longer needs AWS/CA credentials. - New materialize_workspace_deps.py pins bare workspace dependencies to >= floors before publishing (uv drops workspace sources at build time and has no first-class equivalent). - release-trigger also watches __about__.py version files. - docs/versioning.md + CONTRIBUTING.md updated for the new scheme. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/actions/compute-version/action.yml | 118 ++++-------------- .../workflows/compute-versions-dry-run.yaml | 28 ++--- .github/workflows/release-trigger.yaml | 11 +- .../workflows/scripts/detect_version_bumps.py | 99 ++++++++++----- .../scripts/materialize_workspace_deps.py | 106 ++++++++++++++++ CONTRIBUTING.md | 35 +++--- docs/versioning.md | 52 +++++--- 7 files changed, 272 insertions(+), 177 deletions(-) create mode 100644 .github/workflows/scripts/materialize_workspace_deps.py diff --git a/.github/actions/compute-version/action.yml b/.github/actions/compute-version/action.yml index f87b677ee..475ffc902 100644 --- a/.github/actions/compute-version/action.yml +++ b/.github/actions/compute-version/action.yml @@ -1,16 +1,20 @@ name: Compute package version description: > - Computes the version string for a package given branch context. + Computes the internal build version string for a package on a no-bump merge. - Contexts: - - `vnext`: `+dev.` (PEP 440 local version). - Falls back to `..0+dev.` if never published. - Local versions are rejected by PyPI — only suitable for private indexes - like CodeArtifact. - - `main`: `..` — increments the highest - published patch for the same major.minor series. - - `main-bump`: `..0` — used when a major/minor bump commit - lands on main (patch resets to 0). + All three released version components (..) are + human-owned in pyproject.toml; releases publish that version as-is. This + action only versions the interim internal builds published to CodeArtifact + between releases, using a PEP 440 post-release so they order AFTER the + released version and are picked up by `>=` specifiers: + + - `main`: `.post+main.` + - `vnext`: `.post+vnext.` + + The `+main`/`+vnext` local label distinguishes the two build streams; local + labels are ignored during version comparison so ordering comes from + `.post` alone. Local versions are rejected by public PyPI, which + guarantees these builds stay internal. Prerequisites: repo must be checked out and `uv` must be available. @@ -20,14 +24,8 @@ inputs: required: true context: description: > - Branch context controlling the version formula. - Supported values: `vnext`, `main`, `main-bump`. - required: true - index_url: - description: > - PyPI simple index URL with embedded credentials for querying - CodeArtifact. Obtain via the `.github/actions/code-artifact` action's - `index_url` output after configuring AWS credentials. + Branch context naming the build stream. Supported values: `main`, + `vnext`. required: true outputs: @@ -44,89 +42,23 @@ runs: env: PACKAGE: ${{ inputs.package }} CONTEXT: ${{ inputs.context }} - INDEX_URL: ${{ inputs.index_url }} RUN_NUMBER: ${{ github.run_number }} + SHA: ${{ github.sha }} run: | set -euo pipefail - # --- Read seed version from pyproject.toml --- - SEED=$(cd "packages/${PACKAGE}" && uv version --short) - MAJOR_MINOR=$(echo "$SEED" | grep -oE '^[0-9]+\.[0-9]+') - echo "Seed version for ${PACKAGE}: ${SEED} (major.minor: ${MAJOR_MINOR})" - - # --- Query CodeArtifact for the latest published version --- - # uv pip compile resolves the latest matching version from the index. - # We constrain to the current major.minor series for `main` context. - resolve_latest() { - local constraint="$1" - local output - # uv pip compile exits non-zero both when nothing matches (a normal - # "not published yet" result) and on real failures (network/auth/etc). - # Only treat the former as benign; anything else must surface and fail. - output=$(echo "$constraint" \ - | uv pip compile - --index-url "$INDEX_URL" --no-deps --quiet 2>&1) || { - if echo "$output" | grep -qiE 'no solution found|could not find a version|not found in the package registry'; then - echo "" - return 0 - fi - echo "ERROR: uv pip compile failed for '${constraint}':" >&2 - echo "$output" >&2 - exit 1 - } - echo "$output" | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' | head -1 || true - } - - # --- Compute version based on context --- case "$CONTEXT" in - vnext) - LATEST=$(resolve_latest "$PACKAGE") - if [ -n "$LATEST" ]; then - BASE="$LATEST" - echo "Latest published version: ${LATEST}" - else - # No published version at all — use the pyproject.toml seed - # as-is (not just its major.minor) so a patch already bumped - # there (e.g. baselining) isn't regressed. - BASE="$SEED" - echo "No published version found — falling back to seed version ${BASE}" - fi - VERSION="${BASE}+dev.${RUN_NUMBER}" - ;; - - main) - # Resolve the highest patch within the current major.minor series. - LATEST_IN_SERIES=$(resolve_latest "${PACKAGE}>=${MAJOR_MINOR}.0,<${MAJOR_MINOR}.99999") - SEED_PATCH=$(echo "$SEED" | grep -oE '[0-9]+$') - if [ -n "$LATEST_IN_SERIES" ]; then - CURRENT_PATCH=$(echo "$LATEST_IN_SERIES" | grep -oE '[0-9]+$') - NEXT_FROM_PUBLISHED=$((CURRENT_PATCH + 1)) - echo "Latest in ${MAJOR_MINOR}.x series: ${LATEST_IN_SERIES} → next patch: ${NEXT_FROM_PUBLISHED}" - else - NEXT_FROM_PUBLISHED=0 - echo "No published version in ${MAJOR_MINOR}.x series" - fi - # The pyproject.toml seed's patch acts as a floor so a manual - # bump there (e.g. baselining) is never regressed — CI only - # takes over incrementing once publishing has caught up to it. - if [ "$SEED_PATCH" -gt "$NEXT_FROM_PUBLISHED" ]; then - NEXT_PATCH=$SEED_PATCH - echo "Seed patch (${SEED_PATCH}) is ahead of published — using it as the baseline" - else - NEXT_PATCH=$NEXT_FROM_PUBLISHED - fi - VERSION="${MAJOR_MINOR}.${NEXT_PATCH}" - ;; - - main-bump) - VERSION="${MAJOR_MINOR}.0" - echo "Major/minor bump — patch resets to 0" - ;; - + main|vnext) ;; *) - echo "::error::Unknown context '${CONTEXT}'. Supported: vnext, main, main-bump." + echo "::error::Unknown context '${CONTEXT}'. Supported: main, vnext." exit 1 ;; esac + SEED=$(cd "packages/${PACKAGE}" && uv version --short) + SHORT_SHA=$(echo "$SHA" | cut -c1-7) + + VERSION="${SEED}.post${RUN_NUMBER}+${CONTEXT}.${SHORT_SHA}" + echo "Computed version for ${PACKAGE} (${CONTEXT}): ${VERSION}" - echo "version=${VERSION}" >> "$GITHUB_OUTPUT" + echo "version=${VERSION}" >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/compute-versions-dry-run.yaml b/.github/workflows/compute-versions-dry-run.yaml index 3ca263342..fcd8aa716 100644 --- a/.github/workflows/compute-versions-dry-run.yaml +++ b/.github/workflows/compute-versions-dry-run.yaml @@ -2,8 +2,8 @@ name: Compute versions (dry run) # Runs on pushes to vnext and main. Computes and logs the version that would # be published for each affected package — but does not build or publish. -# Also runs (read-only) on PRs that touch the compute-version/code-artifact -# composite actions or this workflow itself, as a smoke test for those. +# Also runs (read-only) on PRs that touch the compute-version composite +# action or this workflow itself, as a smoke test for those. # Remove this workflow once Phase 3 publish workflows are live (see # https://github.com/OvertureMaps/schema/issues/509). @@ -14,20 +14,19 @@ on: branches: [main, vnext] paths: - '**/pyproject.toml' - # Test usage: smoke-tests the compute-version/code-artifact composite - # actions (and this workflow) against a placeholder context on PRs that - # touch them, so wiring regressions surface before merge. + # Test usage: smoke-tests the compute-version composite action (and this + # workflow) against a placeholder context on PRs that touch them, so wiring + # regressions surface before merge. pull_request: paths: - '.github/actions/compute-version/**' - - '.github/actions/code-artifact/**' - '.github/workflows/compute-versions-dry-run.yaml' workflow_dispatch: inputs: context: description: "Version context to simulate" type: choice - options: [vnext, main, main-bump] + options: [vnext, main] default: vnext permissions: @@ -76,10 +75,9 @@ jobs: compute-versions: name: Compute version (${{ matrix.package }})${{ github.event_name == 'pull_request' && ' (test)' || '' }} needs: discover - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: contents: read - id-token: write # Required for OIDC authentication to AWS strategy: fail-fast: false matrix: @@ -96,17 +94,6 @@ jobs: with: persist-credentials: false - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 - with: - aws-region: us-west-2 - role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_ReadOnly - role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} - - - name: Get CodeArtifact credentials - id: ca - uses: ./.github/actions/code-artifact - # Delegates to the same composite action Phase 3 publish workflows will # use, so the version formula only has one implementation to keep correct. - name: Compute version @@ -115,7 +102,6 @@ jobs: with: package: ${{ matrix.package }} context: ${{ needs.discover.outputs.context }} - index_url: ${{ steps.ca.outputs.index_url }} - name: Report computed version env: diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 1ef112aae..3dbfef4cd 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -1,8 +1,8 @@ name: Release trigger # Runs on every push to main that touches any package's pyproject.toml. -# For each package whose . was bumped, cuts a published GitHub -# Release tagged `-v..0`, titled `` `` ``, +# For each package whose .. was bumped, cuts a published +# GitHub Release tagged `-v`, titled `` `` ``, # with notes taken from that package's CHANGELOG.md section. # # Packages version independently (see docs/versioning.md). The umbrella @@ -12,8 +12,9 @@ name: Release trigger # `uvx towncrier build`, folding that package's changelog.d/ fragments into its # CHANGELOG.md (reviewed in the PR). This workflow reads back the section. # -# Patch-only changes are ignored: the patch component is computed by CI (see -# .github/actions/compute-version), not by this workflow. +# All three version components are human-owned; any increase (patch included) +# is a release. No-bump merges publish internal `.postN` builds instead (see +# .github/actions/compute-version). # # NOTE: releases are created with GITHUB_TOKEN, which by design does NOT trigger # further workflow runs. The Phase 3 publish workflow (#509) must therefore be @@ -25,6 +26,8 @@ on: branches: [main] paths: - packages/*/pyproject.toml + # Packages with a hatch dynamic version bump their version file instead. + - packages/*/**/__about__.py permissions: contents: read diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/workflows/scripts/detect_version_bumps.py index 7696a60e2..5c7d9132c 100644 --- a/.github/workflows/scripts/detect_version_bumps.py +++ b/.github/workflows/scripts/detect_version_bumps.py @@ -1,13 +1,13 @@ #!/usr/bin/env python3 """ -Detect per-package . version bumps between two commits on main. +Detect per-package version bumps between two commits on main. Run from the repository root by the `Release trigger` workflow. Compares each `packages/*/pyproject.toml` at the pushed commit (the working tree) against its -content at the `before` commit, and records the packages whose . -increased. Patch-only changes are ignored: the patch component is computed by -CI, not by humans (see docs/versioning.md). +content at the `before` commit, and records the packages whose +`..` increased. All three components are human-owned +(see docs/versioning.md); any increase, including patch-only, cuts a release. Environment: BEFORE The `before` commit SHA (github.event.before). @@ -20,39 +20,63 @@ Exit status: 0 Success (including the no-bump case). - 1 A package's . went backwards; that must never land on main. + 1 A package's version went backwards; that must never land on main. """ +from collections.abc import Callable from pathlib import Path import json import os +import re import subprocess import sys import tomllib +VERSION_ASSIGNMENT = re.compile(rb"""^__version__\s*=\s*["']([^"']+)["']""", re.MULTILINE) -def major_minor(blob: bytes) -> tuple[int, int]: - """Parse `project.version` from pyproject.toml bytes into (major, minor).""" - version = str(tomllib.loads(blob.decode("utf-8"))["project"]["version"]) - major, minor, *_ = version.split(".") - return int(major), int(minor) +def semver( + pyproject_blob: bytes, + package_dir: str, + read_file: Callable[[str], bytes | None], +) -> tuple[int, int, int] | None: + """ + Resolve a package's (major, minor, patch) from its pyproject.toml bytes. -def before_major_minor(before: str, pyproject: str) -> tuple[int, int] | None: + Static `project.version` is read directly. A hatch dynamic version is + resolved by reading the declared version file through `read_file`, which + takes a repo-relative posix path and returns its bytes (or None if + unreadable, in which case the version is unresolvable). + """ + doc = tomllib.loads(pyproject_blob.decode("utf-8")) + if "version" in doc["project"]: + version = str(doc["project"]["version"]) + else: + version_path = f'{package_dir}/{doc["tool"]["hatch"]["version"]["path"]}' + content = read_file(version_path) + if content is None: + return None + match = VERSION_ASSIGNMENT.search(content) + if not match: + return None + version = match.group(1).decode("utf-8") + major, minor, patch, *_ = version.split(".") + return int(major), int(minor), int(patch) + + +def git_show(commit: str, path: str) -> bytes | None: """ - Return the (major, minor) of `pyproject` at the `before` commit. + Return the bytes of `path` at `commit`, or None if unreadable. - The `before` blob can be unreadable after a force-push, a history rewrite, - or for a brand-new package. Treat that as "no previous version" rather than - failing every subsequent push. + A blob can be unreadable after a force-push, a history rewrite, or for a + brand-new file. Treat that as "no previous version" rather than failing + every subsequent push. """ result = subprocess.run( - ["git", "show", f"{before}:{pyproject}"], + ["git", "show", f"{commit}:{path}"], capture_output=True, ) - if result.returncode != 0: - return None - return major_minor(result.stdout) + return result.stdout if result.returncode == 0 else None def main() -> None: @@ -61,32 +85,49 @@ def main() -> None: bumps: list[dict[str, str]] = [] errors: list[str] = [] + def read_working_tree(path: str) -> bytes | None: + try: + return Path(path).read_bytes() + except OSError: + return None + for path in sorted(Path("packages").glob("*/pyproject.toml")): package = path.parent.name - pyproject = path.as_posix() # git wants forward slashes on every OS + package_dir = path.parent.as_posix() # git wants forward slashes on every OS + pyproject = path.as_posix() - after = major_minor(path.read_bytes()) + after = semver(path.read_bytes(), package_dir, read_working_tree) + if after is None: + print(f"Cannot resolve version for {package} in the working tree, skipping.") + continue - current = before_major_minor(before, pyproject) + before_blob = git_show(before, pyproject) + current = ( + semver(before_blob, package_dir, lambda p: git_show(before, p)) + if before_blob is not None + else None + ) if current is None: - print(f"No readable {pyproject} at {before}, skipping {package}.") + print(f"No resolvable version for {package} at {before}, skipping.") continue if after == current: continue + before_str = ".".join(map(str, current)) + after_str = ".".join(map(str, after)) + if after < current: - errors.append(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]}") + errors.append(f"{package}: {before_str} -> {after_str}") continue - version = f"{after[0]}.{after[1]}.0" - print(f"{package}: {current[0]}.{current[1]} -> {after[0]}.{after[1]} (bump)") - bumps.append({"package": package, "version": version, "tag": f"{package}-v{version}"}) + print(f"{package}: {before_str} -> {after_str} (bump)") + bumps.append({"package": package, "version": after_str, "tag": f"{package}-v{after_str}"}) if errors: for e in errors: print( - f"::error::{e}: major/minor went backwards. Version decreases " + f"::error::{e}: version went backwards. Version decreases " "must never land on main; revert or fix the version." ) sys.exit(1) @@ -96,7 +137,7 @@ def main() -> None: out.write(f"bumps={json.dumps(bumps)}\n") if not bumps: - print("No major/minor bumps detected.") + print("No version bumps detected.") if __name__ == "__main__": diff --git a/.github/workflows/scripts/materialize_workspace_deps.py b/.github/workflows/scripts/materialize_workspace_deps.py new file mode 100644 index 000000000..21835a122 --- /dev/null +++ b/.github/workflows/scripts/materialize_workspace_deps.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "tomlkit>=0.13", +# ] +# /// + +""" +Materialize workspace dependency versions into a package's pyproject.toml. + +Run from the repository root, before building a package for publishing: + + uv run ./.github/workflows/scripts/materialize_workspace_deps.py + +Intra-repo dependencies are declared as bare names (e.g. "overture-schema-common") +and resolved at dev time through `[tool.uv.sources]` workspace entries. Built +distributions carry the declared metadata as-is, so without this step a published +wheel would depend on an unconstrained package name. uv has no first-class +feature for this (workspace sources are dev-only and dropped at build time). + +For each dependency that names a workspace member, this script rewrites the +bare name to a floor constraint from the version currently in the repo, e.g. +"overture-schema-common" becomes "overture-schema-common>=0.1.1". The floor is +the released version only; internal `.postN` build suffixes are never written +into dependency constraints. Edits preserve pyproject.toml formatting. + +Dependencies that already carry a constraint, and dependencies on packages +outside this repo, are left untouched. + +Exit status: + 0 Success (including nothing-to-do). + 1 Unknown package, or a workspace dependency's pyproject cannot be read. +""" + +from pathlib import Path +import re +import sys + +import tomlkit + +PACKAGES_DIR = Path("packages") + +# A bare PEP 508 name: no extras, no specifier, no markers. +BARE_NAME = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$") + +VERSION_ASSIGNMENT = re.compile(r"""^__version__\s*=\s*["']([^"']+)["']""", re.MULTILINE) + + +def dynamic_version(package_dir: Path, doc: tomlkit.TOMLDocument) -> str: + """Resolve a hatch dynamic version from its declared version file.""" + version_path = str(doc["tool"]["hatch"]["version"]["path"]) + content = (package_dir / version_path).read_text(encoding="utf-8") + match = VERSION_ASSIGNMENT.search(content) + if not match: + raise ValueError(f"No __version__ assignment in {package_dir / version_path}") + return match.group(1) + + +def workspace_versions() -> dict[str, str]: + """Map every workspace member's distribution name to its in-repo version.""" + versions: dict[str, str] = {} + for path in sorted(PACKAGES_DIR.glob("*/pyproject.toml")): + doc = tomlkit.parse(path.read_text(encoding="utf-8")) + project = doc["project"] + if "version" in project: + version = str(project["version"]) + else: + version = dynamic_version(path.parent, doc) + versions[str(project["name"])] = version + return versions + + +def materialize(package: str) -> int: + pyproject_path = PACKAGES_DIR / package / "pyproject.toml" + if not pyproject_path.is_file(): + print(f"::error::No such package: {package} ({pyproject_path} missing).") + return 1 + + versions = workspace_versions() + doc = tomlkit.parse(pyproject_path.read_text(encoding="utf-8")) + dependencies = doc["project"].get("dependencies", []) + + changed = 0 + for i, dep in enumerate(dependencies): + name = str(dep).strip() + if not BARE_NAME.match(name) or name not in versions: + continue + floor = f"{name}>={versions[name]}" + dependencies[i] = floor + changed += 1 + print(f"{package}: {name} -> {floor}") + + if changed: + pyproject_path.write_text(tomlkit.dumps(doc), encoding="utf-8") + print(f"{package}: materialized {changed} workspace dependenc{'y' if changed == 1 else 'ies'}.") + else: + print(f"{package}: no bare workspace dependencies to materialize.") + return 0 + + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("usage: materialize_workspace_deps.py ", file=sys.stderr) + sys.exit(1) + sys.exit(materialize(sys.argv[1])) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index affb5f2c3..1d066bd63 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,13 +22,14 @@ wiki page breaks down what counts as a minor vs. major change. Three common paths take a branch to a merge. Expand each for the commit-level flow.
-main → patch (no version bump) +main → everyday change (no version bump) -Everyday bug fixes and schema tweaks. You do not touch the version; on merge CI -publishes the next patch (`..`) to internal -CodeArtifact, where it is consumable immediately. No GitHub Release is cut and -nothing new lands on public PyPI; the patch reaches public PyPI only when the -next minor or major release ships. +Everyday fixes and schema tweaks that don't warrant an immediate release. You +do not touch the version; on merge CI publishes an interim internal build +(`.postN+main.`) to CodeArtifact, where it is consumable +immediately. No GitHub Release is cut and nothing lands on public PyPI; the +change reaches public PyPI with the package's next version bump (patch, minor, +or major). ```mermaid gitGraph @@ -38,18 +39,18 @@ gitGraph commit id: "fix brand enum values" commit id: "add bugfix fragment" checkout main - merge fix-places-brand-enum id: "PR #561 (CodeArtifact 0.4.1)" + merge fix-places-brand-enum id: "PR #561 (CodeArtifact 0.4.0.postN)" commit id: "more fixes" ```
-main → minor release (version bump) +main → patch or minor release (version bump) -A minor feature that bumps `major.minor` in the PR and builds the changelog. On -merge, `release-trigger` cuts a published GitHub Release and the new version lands -on PyPI, immediately available to consumers. +A bug fix or minor feature that bumps the version in the PR and builds the +changelog. On merge, `release-trigger` cuts a published GitHub Release and the +new version lands on PyPI, immediately available to consumers. ```mermaid gitGraph @@ -100,7 +101,7 @@ gitGraph The `bump ... + build changelog` commit edits the package version in `pyproject.toml` and folds its `changelog.d/` fragments into `CHANGELOG.md`. On merge to `main`, CI cuts a published GitHub Release tagged -`-v..0` with those notes. See +`-v` with those notes. See [docs/versioning.md](docs/versioning.md). ## Opening a PR @@ -119,10 +120,12 @@ merge to `main`; run `git pull --rebase` before pushing again. ## Changing a package version -- `.` is your call: edit it in `pyproject.toml` and reset patch to - `0` (e.g. `1.17.1` becomes `1.18.0`). Minor bumps target `main`; major bumps - target `vnext`. -- `` is computed by CI at publish time; never edit it manually. +- The full `..` is your call: edit it in `pyproject.toml` + and any increase (patch included) ships a release. Patch and minor bumps + target `main`; major bumps target `vnext`. +- Between releases, CI stamps interim internal builds as + `.postN+main.` (or `+vnext.`); never write `.postN` + suffixes manually. - Every package versions and releases independently. Consumers pin only `overture-schema`, which pulls in the theme and support packages for a coherent set. diff --git a/docs/versioning.md b/docs/versioning.md index a32bc4071..79c532b39 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -8,6 +8,7 @@ Reference and how-to for package versions and releases. Branch mechanics and the - [Reference](#reference) - [Version scheme](#version-scheme) - [Version → destination](#version--destination) + - [Dependency materialization](#dependency-materialization) - [Tag scheme](#tag-scheme) - [Guardrails](#guardrails) - [How to](#how-to) @@ -24,16 +25,33 @@ Every distributable package under `packages/*` carries its own independent | Component | Owner | Set by | |-----------|-------|--------| -| `.` | Human | Edited in `pyproject.toml` via a reviewed PR. | -| `` | CI | [`compute-version`](../.github/actions/compute-version/action.yml) at publish time. The `pyproject.toml` patch is only a floor. | +| `..` | Human | Edited in `pyproject.toml` via a reviewed PR. Any increase, patch included, is a release. | +| `.post+.` | CI | [`compute-version`](../.github/actions/compute-version/action.yml) stamps interim internal builds between releases. | ### Version → destination | Event | Version | Destination | |-------|---------|-------------| -| Push to `vnext` | `+dev.` | CodeArtifact (dev) | -| Push to `main`, no bump | `..` | CodeArtifact | -| `major.minor` bump on `main` | `..0` | Public PyPI | +| Push to `vnext` | `.postN+vnext.` | CodeArtifact | +| Push to `main`, no bump | `.postN+main.` | CodeArtifact | +| Version bump on `main` | `` | GitHub Release, then public PyPI | + +Internal builds use PEP 440 post-releases so they order after the released +``: a consumer pinning `>=1.2.3` resolves `1.2.3.post4+main.abc1234` +from CodeArtifact when present. `N` is the workflow run number. The +`+main`/`+vnext` local label names the build stream; local labels are ignored +in version comparison and rejected by public PyPI, which keeps these builds +internal by construction. + +### Dependency materialization + +Intra-repo dependencies are declared as bare names and resolved at dev time +via `[tool.uv.sources]` workspace entries, which uv drops at build time. +Before publishing, CI runs +[`materialize_workspace_deps.py`](../.github/workflows/scripts/materialize_workspace_deps.py) +to rewrite each bare workspace dependency to a floor from the in-repo version +(e.g. `overture-schema-common` becomes `overture-schema-common>=0.1.1`). +Floors carry the released version only, never a `.postN` suffix. ### Tag scheme @@ -78,7 +96,7 @@ The file body is the note itself, written in past tense ```bash # from the repo root -uvx towncrier build --config pyproject.toml --dir packages/ --draft --version ..0 +uvx towncrier build --config pyproject.toml --dir packages/ --draft --version ``` A fragment (or an already-built `CHANGELOG.md` entry) is required on any PR that @@ -91,13 +109,13 @@ changes that package, whether or not it bumps the version. ### Cut a release -1. Bump `.` in the package's `pyproject.toml` (reset patch to `0`), +1. Bump the version in the package's `pyproject.toml`, then run `uvx towncrier build --config pyproject.toml --dir packages/` - from the repo root to fold its fragments into `CHANGELOG.md`. Minor bumps - target `main`; major bumps go via `vnext` and reach `main` through a release - merge. + from the repo root to fold its fragments into `CHANGELOG.md`. Patch and minor + bumps target `main`; major bumps go via `vnext` and reach `main` through a + release merge. 2. On merge to `main`, `release-trigger` publishes one GitHub Release per bumped - package: tag `-v..0`, notes from that package's + package: tag `-v`, notes from that package's `CHANGELOG.md`. 3. Publishing the release starts the PyPI publish, gated by a maintainer approval. @@ -106,13 +124,19 @@ changes that package, whether or not it bumps the version. flowchart LR A[bump + towncrier build
merged to main] --> B[release-trigger:
GitHub Release per package] B --> C[PyPI publish
maintainer approval] --> D[public PyPI] - E[patch to main] --> F[CodeArtifact only] + E[no-bump merge] --> F[.postN internal build
CodeArtifact only] ``` ## Why -- **Human owns `major.minor`, CI owns `patch`.** Release intent is a reviewed - decision; patch numbering is mechanical. +- **Humans own the full version.** Every released `..` is + a reviewed decision, so patch-level bug fixes can ship to PyPI without + masquerading as minor releases. CI versions only the interim `.postN` + builds between releases. +- **Post-releases for internal builds.** `.postN` orders after the released + version, so `>=` specifiers pick up the freshest internal build + from CodeArtifact; the `+main`/`+vnext` label separates the two streams and + keeps the builds off public PyPI. - **Independent per-package versions.** Packages evolve at their own pace. Consumers pin only `overture-schema`, which depends on the theme/support packages, giving them a coherent set without tracking each one. From 248ac096f42f9256fab7f6331a77296c168a8899 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 15:03:45 -0400 Subject: [PATCH 16/41] [REFACTOR](ci) Drop dynamic-version handling; all packages are static now Upstream baselined overture-schema-pyspark to a static version (a3303be5), so the hatch dynamic-version resolution in detect_version_bumps.py and materialize_workspace_deps.py is dead code, as is the __about__.py path trigger on release-trigger. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/release-trigger.yaml | 2 - .../workflows/scripts/detect_version_bumps.py | 56 +++---------------- .../scripts/materialize_workspace_deps.py | 21 +------ 3 files changed, 10 insertions(+), 69 deletions(-) diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 3dbfef4cd..162b721f7 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -26,8 +26,6 @@ on: branches: [main] paths: - packages/*/pyproject.toml - # Packages with a hatch dynamic version bump their version file instead. - - packages/*/**/__about__.py permissions: contents: read diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/workflows/scripts/detect_version_bumps.py index 5c7d9132c..ecb923cd2 100644 --- a/.github/workflows/scripts/detect_version_bumps.py +++ b/.github/workflows/scripts/detect_version_bumps.py @@ -23,43 +23,17 @@ 1 A package's version went backwards; that must never land on main. """ -from collections.abc import Callable from pathlib import Path import json import os -import re import subprocess import sys import tomllib -VERSION_ASSIGNMENT = re.compile(rb"""^__version__\s*=\s*["']([^"']+)["']""", re.MULTILINE) - -def semver( - pyproject_blob: bytes, - package_dir: str, - read_file: Callable[[str], bytes | None], -) -> tuple[int, int, int] | None: - """ - Resolve a package's (major, minor, patch) from its pyproject.toml bytes. - - Static `project.version` is read directly. A hatch dynamic version is - resolved by reading the declared version file through `read_file`, which - takes a repo-relative posix path and returns its bytes (or None if - unreadable, in which case the version is unresolvable). - """ - doc = tomllib.loads(pyproject_blob.decode("utf-8")) - if "version" in doc["project"]: - version = str(doc["project"]["version"]) - else: - version_path = f'{package_dir}/{doc["tool"]["hatch"]["version"]["path"]}' - content = read_file(version_path) - if content is None: - return None - match = VERSION_ASSIGNMENT.search(content) - if not match: - return None - version = match.group(1).decode("utf-8") +def semver(pyproject_blob: bytes) -> tuple[int, int, int]: + """Parse `project.version` from pyproject.toml bytes into (major, minor, patch).""" + version = str(tomllib.loads(pyproject_blob.decode("utf-8"))["project"]["version"]) major, minor, patch, *_ = version.split(".") return int(major), int(minor), int(patch) @@ -85,31 +59,17 @@ def main() -> None: bumps: list[dict[str, str]] = [] errors: list[str] = [] - def read_working_tree(path: str) -> bytes | None: - try: - return Path(path).read_bytes() - except OSError: - return None - for path in sorted(Path("packages").glob("*/pyproject.toml")): package = path.parent.name - package_dir = path.parent.as_posix() # git wants forward slashes on every OS - pyproject = path.as_posix() + pyproject = path.as_posix() # git wants forward slashes on every OS - after = semver(path.read_bytes(), package_dir, read_working_tree) - if after is None: - print(f"Cannot resolve version for {package} in the working tree, skipping.") - continue + after = semver(path.read_bytes()) before_blob = git_show(before, pyproject) - current = ( - semver(before_blob, package_dir, lambda p: git_show(before, p)) - if before_blob is not None - else None - ) - if current is None: - print(f"No resolvable version for {package} at {before}, skipping.") + if before_blob is None: + print(f"No readable {pyproject} at {before}, skipping {package}.") continue + current = semver(before_blob) if after == current: continue diff --git a/.github/workflows/scripts/materialize_workspace_deps.py b/.github/workflows/scripts/materialize_workspace_deps.py index 21835a122..f4459ce86 100644 --- a/.github/workflows/scripts/materialize_workspace_deps.py +++ b/.github/workflows/scripts/materialize_workspace_deps.py @@ -44,30 +44,13 @@ # A bare PEP 508 name: no extras, no specifier, no markers. BARE_NAME = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$") -VERSION_ASSIGNMENT = re.compile(r"""^__version__\s*=\s*["']([^"']+)["']""", re.MULTILINE) - - -def dynamic_version(package_dir: Path, doc: tomlkit.TOMLDocument) -> str: - """Resolve a hatch dynamic version from its declared version file.""" - version_path = str(doc["tool"]["hatch"]["version"]["path"]) - content = (package_dir / version_path).read_text(encoding="utf-8") - match = VERSION_ASSIGNMENT.search(content) - if not match: - raise ValueError(f"No __version__ assignment in {package_dir / version_path}") - return match.group(1) - def workspace_versions() -> dict[str, str]: """Map every workspace member's distribution name to its in-repo version.""" versions: dict[str, str] = {} for path in sorted(PACKAGES_DIR.glob("*/pyproject.toml")): - doc = tomlkit.parse(path.read_text(encoding="utf-8")) - project = doc["project"] - if "version" in project: - version = str(project["version"]) - else: - version = dynamic_version(path.parent, doc) - versions[str(project["name"])] = version + project = tomlkit.parse(path.read_text(encoding="utf-8"))["project"] + versions[str(project["name"])] = str(project["version"]) return versions From 40462eac67766407d5f6139faf97ceffbc4d6e37 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 15:22:44 -0400 Subject: [PATCH 17/41] [REFACTOR](ci) Diff package versions from git blobs instead of dual checkout Replaces package-versions.py (installed-metadata collection with a hardcoded topology map its own comment called brittle) with package_versions.py, which reads pyproject.toml blobs straight from git at both commits and derives the topological order dynamically from each package's declared dependencies. The reusable version-check workflow drops its second checkout, both uv sync steps, and the Python/uv setup on the no-change path - the diff now needs only a fetch-depth: 0 checkout and the standard library. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- ...eusable-check-python-package-versions.yaml | 77 +++------ .github/workflows/scripts/package-versions.py | 154 ------------------ .github/workflows/scripts/package_versions.py | 109 +++++++++++++ 3 files changed, 135 insertions(+), 205 deletions(-) delete mode 100755 .github/workflows/scripts/package-versions.py create mode 100644 .github/workflows/scripts/package_versions.py diff --git a/.github/workflows/reusable-check-python-package-versions.yaml b/.github/workflows/reusable-check-python-package-versions.yaml index 93e81644d..fbea4506a 100644 --- a/.github/workflows/reusable-check-python-package-versions.yaml +++ b/.github/workflows/reusable-check-python-package-versions.yaml @@ -1,5 +1,9 @@ name: "[REUSABLE] Check Python package versions" +# Diffs packages/*/pyproject.toml versions between two commits, entirely from +# git blobs (single checkout, no environment sync), and verifies that any new +# version does not already exist in CodeArtifact. + on: workflow_call: inputs: @@ -67,64 +71,31 @@ jobs: changed_packages: ${{ steps.save-changes.outputs.changed_packages }} num_changed_packages: ${{ steps.save-changes.outputs.num_changed_packages }} steps: - - name: Install jq - run: sudo apt-get update && sudo apt-get install -y jq - - - name: Install uv - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - with: - version: latest - - - name: Check out code before change + - name: Check out code uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - ref: ${{ inputs.before_commit }} + fetch-depth: 0 # Both comparison commits must be reachable persist-credentials: false - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version-file: .python-version - - - name: Sync code before change to make packages visible to Python - run: uv sync --all-packages - - - name: Capture package versions before change - run: uv run python ./.github/workflows/scripts/package-versions.py collect > /tmp/package-versions-before.json - - - name: Check out code after change - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - ref: ${{ inputs.after_commit }} - persist-credentials: false - - - name: Sync code after change to make packages visible to Python - run: uv sync --all-packages --refresh - - - name: Capture package versions after change - run: uv run python ./.github/workflows/scripts/package-versions.py collect > /tmp/package-versions-after.json - - - name: Compare package versions before and after change - run: | - uv run python ./.github/workflows/scripts/package-versions.py compare \ - /tmp/package-versions-before.json \ - /tmp/package-versions-after.json \ - >/tmp/package-version-diff.json - - - name: Print changed versions - run: cat /tmp/package-version-diff.json - - - name: Save changed versions as output + - name: Diff package versions id: save-changes + env: + BEFORE: ${{ inputs.before_commit }} + AFTER: ${{ inputs.after_commit }} run: | - echo 'changed_packages<> $GITHUB_OUTPUT - cat /tmp/package-version-diff.json >> $GITHUB_OUTPUT - echo EOF >> $GITHUB_OUTPUT - printf 'num_changed_packages=%s\n' "$(jq -c '. | length' /tmp/package-version-diff.json)" >> $GITHUB_OUTPUT + python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" "$AFTER" \ + > /tmp/package-version-diff.json + cat /tmp/package-version-diff.json + { + echo 'changed_packages<> "$GITHUB_OUTPUT" - name: Configure AWS credentials if: steps.save-changes.outputs.num_changed_packages > 0 - uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 with: aws-region: ${{ inputs.aws_region }} role-to-assume: arn:aws:iam::${{ inputs.aws_account_id }}:role/${{ inputs.aws_iam_role_name }} @@ -149,8 +120,12 @@ jobs: jq -c '.[]' /tmp/package-version-diff.json | while read -r entry; do package=$(echo "$entry" | jq -r '.package') after=$(echo "$entry" | jq -r '.after') + if [ "$after" = "null" ]; then + echo "Package ${package} was removed. Skipping existence check." + continue + fi exit_code=0 - output=$(uv run pip download "${package}==${after}" --index-url "${INDEX_URL}" --no-deps -d /tmp --quiet 2>&1) || exit_code=$? + output=$(python3 -m pip download "${package}==${after}" --index-url "${INDEX_URL}" --no-deps -d /tmp --quiet 2>&1) || exit_code=$? if [[ $exit_code -eq 0 || ( "${output,,}" != *"could not find a version"* && "${output,,}" != *"no matching distributions"* @@ -162,4 +137,4 @@ jobs: else echo "Package ${package} version ${after} is new, as expected. Continuing." fi - done + done \ No newline at end of file diff --git a/.github/workflows/scripts/package-versions.py b/.github/workflows/scripts/package-versions.py deleted file mode 100755 index a0080f95d..000000000 --- a/.github/workflows/scripts/package-versions.py +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/env python3 - -from importlib import metadata -from pathlib import Path -import json -import re -import sys - - -def collect(): - """ - Collect Python package versions and print them as a JSON array. - - Form of the JSON array: - - [ {"package": "p1", "version": "v1"}, {"package": "p2", "version": "v2"}, ... ] - """ - packages_dir = Path("packages") - - packages = sorted( - d.name - for d in packages_dir.iterdir() - if d.is_dir() and d.name.startswith("overture-schema") and (d / "pyproject.toml").exists() - ) - - package_versions = [ - {"package": p, "version": metadata.version(p.replace("-", "."))} - for p in packages - ] - - print(json.dumps(package_versions, indent=2)) - - -def compare(before_file: str, after_file: str): - """ - Compare two JSON files containing package versions and print the packages that have a version - number change as a JSON array. - - The output JSON array is sorted in topological order by package name, so those changed packages - that do not depend on other changed packages appear first. - - Form of the JSON array: - - [ {"package": "p1", "before": "v1", "after": "v2"}, ... ] - - Note that `before` will be `null` if the package did not exist in the "before" file, and `after` - will be `null` if the package did not exist in the "after" file. - """ - before_array = load(before_file) - after_array = load(after_file) - - before_dict = {item["package"]: item["version"] for item in before_array} - after_dict = {item["package"]: item["version"] for item in after_array} - - def level(package: str) -> int: - """ - Return the level of a package for topological sorting. - - This is brittle and hard to keep in sync, so we should replace it with a version that - dynamically computes dependencies in the future. - """ - if package == "overture-schema-system": - return 0 - elif package in ["overture-schema-common", "overture-schema-core"]: - return 1 - elif re.fullmatch(r'overture-schema-.*-theme', package) or package in ["overture-schema", "overture-schema-cli", "overture-schema-codegen", "overture-schema-annex", "overture-schema-pyspark"]: - return 2 - else: - raise ValueError(f"Unknown package for level computation: {package}") - - combined_keys = sorted(list(set(before_dict.keys()) | set(after_dict.keys())), key=level) - - changed_packages = [] - for package in combined_keys: - before_version = before_dict.get(package) - after_version = after_dict.get(package) - if before_version != after_version: - changed_packages.append( - { - "package": package, - "before": before_version, - "after": after_version, - } - ) - - print(json.dumps(changed_packages, indent=2)) - - -def load(file_path: str) -> list[dict[str, str]]: - path = Path(file_path) - if not path.exists(): - print(f"File not found: {file_path}") - sys.exit(1) - - with path.open() as f: - value = json.load(f) - - if not isinstance(value, list): - print( - f"File {file_path} contains unexpected root value: expected a `list` but got value {repr(value)} of type `{type(value).__name__}`" - ) - sys.exit(1) - - for i, item in enumerate(value): - if not isinstance(item, dict): - print( - f"File {file_path} contains unexpected item at index {i}: expected `dict` but got value {repr(item)} of type `{type(item).__name__}`" - ) - sys.exit(1) - elif sorted(item.keys()) != ["package", "version"]: - print( - f"File {file_path} contains unexpected item at index {i}: expected keys `['package', 'version']` but got keys {sorted(item.keys())}" - ) - sys.exit(1) - elif not isinstance(item["package"], str): - print( - f"File {file_path} contains unexpected item at index {i}: expected `package` to be of type `str` but got value {repr(item['package'])} of type `{type(item['package']).__name__}`" - ) - sys.exit(1) - elif not isinstance(item["version"], str): - print( - f"File {file_path} contains unexpected item at index {i}: expected `version` to be of type `str` but got value {repr(item['version'])} of type `{type(item['version']).__name__}`" - ) - sys.exit(1) - - return value - - -def usage(): - print("Usage:") - print(f" ./{sys.argv[0]} collect") - print(f" ./{sys.argv[0]} compare BEFORE_FILE AFTER_FILE") - sys.exit(1) - - -def main(): - if len(sys.argv) < 2: - usage() - - cmd = sys.argv[1] - - if cmd == "collect": - collect() - elif cmd == "compare": - if len(sys.argv) != 4: - usage() - compare(sys.argv[2], sys.argv[3]) - else: - print(f"Unknown command: {cmd}") - usage() - - -if __name__ == "__main__": - main() diff --git a/.github/workflows/scripts/package_versions.py b/.github/workflows/scripts/package_versions.py new file mode 100644 index 000000000..d4865a5c1 --- /dev/null +++ b/.github/workflows/scripts/package_versions.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +""" +Diff per-package versions between two git commits. + +Run from the repository root: + + python3 package_versions.py diff + +Reads each `packages/*/pyproject.toml` blob directly from git at both commits +(no checkout switching, no environment sync) and prints the packages whose +version changed as a JSON array, topologically sorted so that packages with no +changed dependencies come first. The dependency order is derived from each +package's declared `project.dependencies`, restricted to workspace members. + +Form of the JSON array: + + [ {"package": "p1", "before": "v1", "after": "v2"}, ... ] + +`before` is null if the package did not exist at the before commit; `after` is +null if it no longer exists at the after commit. + +Exit status: + 0 Success. + 1 Usage error. +""" + +from graphlib import TopologicalSorter +import json +import re +import subprocess +import sys +import tomllib + +PACKAGES_DIR = "packages" + +# The distribution name at the start of a PEP 508 requirement string. +REQUIREMENT_NAME = re.compile(r"^\s*([A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)") + + +def git(*args: str) -> str: + result = subprocess.run(["git", *args], capture_output=True, check=True) + return result.stdout.decode("utf-8") + + +def package_manifests(commit: str) -> dict[str, dict]: + """Map package directory name -> parsed pyproject.toml at `commit`.""" + try: + listing = git("ls-tree", "--name-only", commit, f"{PACKAGES_DIR}/") + except subprocess.CalledProcessError: + return {} # commit unreadable (force-push) or no packages dir yet + + manifests: dict[str, dict] = {} + for line in listing.splitlines(): + package = line.removeprefix(f"{PACKAGES_DIR}/") + try: + blob = git("show", f"{commit}:{PACKAGES_DIR}/{package}/pyproject.toml") + except subprocess.CalledProcessError: + continue # not a package directory + manifests[package] = tomllib.loads(blob) + return manifests + + +def topo_order(manifests: dict[str, dict]) -> list[str]: + """Package names sorted so dependencies come before their dependents.""" + dist_to_dir = { + str(m["project"]["name"]): package for package, m in manifests.items() + } + graph: dict[str, set[str]] = {} + for package, manifest in manifests.items(): + deps = set() + for requirement in manifest["project"].get("dependencies", []): + match = REQUIREMENT_NAME.match(str(requirement)) + if match and match.group(1) in dist_to_dir: + deps.add(dist_to_dir[match.group(1)]) + graph[package] = deps + return list(TopologicalSorter(graph).static_order()) + + +def diff(before: str, after: str) -> None: + before_manifests = package_manifests(before) + after_manifests = package_manifests(after) + + def version(manifests: dict[str, dict], package: str) -> str | None: + manifest = manifests.get(package) + if manifest is None: + return None + # A dynamic version (no static `project.version`) also maps to None. + value = manifest["project"].get("version") + return str(value) if value is not None else None + + # Order from the after commit, which knows about newly added packages; + # packages that only exist in before (deleted) are appended at the end. + order = topo_order(after_manifests) + order += sorted(set(before_manifests) - set(after_manifests)) + + changed = [ + {"package": p, "before": b, "after": a} + for p in order + if (b := version(before_manifests, p)) != (a := version(after_manifests, p)) + ] + print(json.dumps(changed, indent=2)) + + +if __name__ == "__main__": + if len(sys.argv) != 4 or sys.argv[1] != "diff": + print(f"Usage: {sys.argv[0]} diff BEFORE_COMMIT AFTER_COMMIT", file=sys.stderr) + sys.exit(1) + diff(sys.argv[2], sys.argv[3]) From 1d567617d3689fd93c3aad7462647317bbbb00df Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 15:31:03 -0400 Subject: [PATCH 18/41] [REFACTOR](ci) Replace github-script checks with plain git and bash The changelog fragment check and the change-type label check were the only two workflows still embedding JavaScript via actions/github-script. Both reduce to a git diff / jq over the event payload plus grep, with no action dependency and the same failure messages. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../workflows/enforce-change-type-label.yaml | 51 +++++------ .../workflows/require-changelog-fragment.yaml | 90 +++++++++---------- 2 files changed, 70 insertions(+), 71 deletions(-) diff --git a/.github/workflows/enforce-change-type-label.yaml b/.github/workflows/enforce-change-type-label.yaml index 9f476feec..2264c4652 100644 --- a/.github/workflows/enforce-change-type-label.yaml +++ b/.github/workflows/enforce-change-type-label.yaml @@ -14,34 +14,35 @@ concurrency: jobs: check-label: name: Check label - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: contents: read # Required for reading PR labels steps: - name: Require exactly one change type label - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - with: - script: | - const allChangeTypeLabels = new Set([ - 'change type - cosmetic 🌹', - 'change type - documentation - docs team 📝', - 'change type - documentation - member 📝', - 'change type - major 🚨', - 'change type - minor 🤏', - 'automation 🦾', - ]); - const prLabels = context.payload.pull_request.labels.map(label => label.name); - const appliedChangeTypeLabels = prLabels.filter(prLabel => allChangeTypeLabels.has(prLabel)); - if (appliedChangeTypeLabels.length !== 1) { - const baseMessage = `The PR must have EXACTLY one of the following CHANGE TYPE labels: ${Array.from(allChangeTypeLabels).sort().join(', ')}. ` - const n = appliedChangeTypeLabels.length; - let contextualMessage; - if (n === 0) { - contextualMessage = 'It currently has no change type label. Please ➕ add one label. 🙏' - } else { - contextualMessage = `It currently has ${n} change type labels (${JSON.stringify(appliedChangeTypeLabels)}). 🙏 Please ❌ remove ${n-1} label(s).` - } - core.setFailed(baseMessage + contextualMessage); - } + run: | + set -euo pipefail + + change_type_labels='change type - cosmetic 🌹 + change type - documentation - docs team 📝 + change type - documentation - member 📝 + change type - major 🚨 + change type - minor 🤏 + automation 🦾' + + applied=$(jq -r '.pull_request.labels[].name' "$GITHUB_EVENT_PATH" \ + | grep -Fxf <(echo "$change_type_labels" | sed 's/^[[:space:]]*//') || true) + count=$(echo "$applied" | grep -c . || true) + + if [ "$count" -ne 1 ]; then + echo "::error::The PR must have EXACTLY one change type label; it has ${count}." + echo "Allowed labels:" + echo "$change_type_labels" | sed 's/^[[:space:]]*/ - /' + if [ "$count" -gt 1 ]; then + echo "Currently applied:" + echo "$applied" | sed 's/^/ - /' + fi + exit 1 + fi + echo "Change type label check passed: ${applied}" \ No newline at end of file diff --git a/.github/workflows/require-changelog-fragment.yaml b/.github/workflows/require-changelog-fragment.yaml index d14f0a180..197550e86 100644 --- a/.github/workflows/require-changelog-fragment.yaml +++ b/.github/workflows/require-changelog-fragment.yaml @@ -24,57 +24,55 @@ concurrency: jobs: check-fragment: name: Require changelog on package change - runs-on: ubuntu-latest + runs-on: ubuntu-slim permissions: - contents: read # Read PR file list - pull-requests: read # List PR files + contents: read steps: - - name: Require a changelog update for each changed package - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: - script: | - const { owner, repo } = context.repo; - const pr = context.payload.pull_request; - - const files = await github.paginate(github.rest.pulls.listFiles, { - owner, repo, pull_number: pr.number, per_page: 100, - }); + fetch-depth: 0 # The merge base with the PR base branch must be reachable + persist-credentials: false - const fragmentPattern = pkg => - new RegExp(`^packages/${pkg}/changelog\\.d/.+\\.(breaking|feature|bugfix|docs|misc)\\.md$`); - - // A package "changed" if any of its files changed other than its own - // changelog artifacts (the fragment or the built CHANGELOG.md). - const isChangelogArtifact = (pkg, path) => - path.startsWith(`packages/${pkg}/changelog.d/`) || - path === `packages/${pkg}/CHANGELOG.md`; + - name: Require a changelog update for each changed package + env: + BASE_REF: ${{ github.base_ref }} + run: | + set -euo pipefail - const touched = new Set(); - for (const file of files) { - const match = file.filename.match(/^packages\/([^/]+)\//); - if (!match) continue; - const pkg = match[1]; - if (!isChangelogArtifact(pkg, file.filename)) touched.add(pkg); - } + changed=$(git diff --name-only "origin/${BASE_REF}...HEAD") - const missing = []; - for (const pkg of [...touched].sort()) { - const hasChangelog = files.some(f => f.filename === `packages/${pkg}/CHANGELOG.md`); - const hasFragment = files.some(f => - f.status === 'added' && fragmentPattern(pkg).test(f.filename)); + # Packages with changes outside their own changelog artifacts. + touched=$(echo "$changed" \ + | grep -oE '^packages/[^/]+/' \ + | sort -u \ + | while read -r prefix; do + pkg=$(basename "$prefix") + if echo "$changed" | grep -E "^packages/${pkg}/" \ + | grep -vE "^packages/${pkg}/(changelog\.d/|CHANGELOG\.md$)" \ + | grep -q .; then + echo "$pkg" + fi + done) - if (!hasChangelog && !hasFragment) missing.push(pkg); - } + missing="" + for pkg in $touched; do + has_fragment=$(echo "$changed" \ + | grep -cE "^packages/${pkg}/changelog\.d/.+\.(breaking|feature|bugfix|docs|misc)\.md$" || true) + has_changelog=$(echo "$changed" \ + | grep -cE "^packages/${pkg}/CHANGELOG\.md$" || true) + if [ "$has_fragment" -eq 0 ] && [ "$has_changelog" -eq 0 ]; then + missing="${missing} - ${pkg}"$'\n' + fi + done - if (missing.length > 0) { - core.setFailed( - `These packages changed but include no changelog update:\n` + - missing.map(m => ` - \`${m}\``).join('\n') + `\n\n` + - `Add a towncrier fragment at packages//changelog.d/..md ` + - `(type: breaking | feature | bugfix | docs | misc), then run ` + - `\`uvx towncrier build --config pyproject.toml --dir packages/\` to fold it into CHANGELOG.md. ` + - `See docs/versioning.md.` - ); - } else { - core.info('Changelog fragment check passed.'); - } + if [ -n "$missing" ]; then + echo "::error::These packages changed but include no changelog update:" + printf '%s' "$missing" + echo "" + echo "Add a towncrier fragment at packages//changelog.d/..md" + echo "(type: breaking | feature | bugfix | docs | misc), then run" + echo '`uvx towncrier build --config pyproject.toml --dir packages/`' + echo "to fold it into CHANGELOG.md. See docs/versioning.md." + exit 1 + fi + echo "Changelog fragment check passed." \ No newline at end of file From a9563ceb4f7111bd61cc0ec212801a2fdb8af06e Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 15:37:01 -0400 Subject: [PATCH 19/41] [DOCS](versioning) Note verified resolver behavior and the >= vs > pin rule Local two-wheel resolution test confirms a bare >=X.Y.Z pin selects X.Y.Z.postN+stream.sha while ==X.Y.Z selects the clean release. PEP 440 excludes post-releases from exclusive comparisons, so consumers must pin with >=, never >. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- docs/versioning.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/versioning.md b/docs/versioning.md index 79c532b39..a0ec43b59 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -38,7 +38,10 @@ Every distributable package under `packages/*` carries its own independent Internal builds use PEP 440 post-releases so they order after the released ``: a consumer pinning `>=1.2.3` resolves `1.2.3.post4+main.abc1234` -from CodeArtifact when present. `N` is the workflow run number. The +from CodeArtifact when present (verified with uv), while `==1.2.3` still +selects the clean release. Pin with `>=`, never `>`: PEP 440 excludes +post-releases from exclusive ordered comparisons, so `>1.2.3` matches no +internal build. `N` is the workflow run number. The `+main`/`+vnext` local label names the build stream; local labels are ignored in version comparison and rejected by public PyPI, which keeps these builds internal by construction. From 0d381695de87a2d8ee0f4a6c813e59cd6fb06b53 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 15:50:23 -0400 Subject: [PATCH 20/41] [DOCS](versioning) Block vnext internal builds on a dedicated dev repository Local labels do not participate in PEP 440 ordering, so main and vnext post-builds sharing one CodeArtifact repository would let a >= consumer resolve a vnext build (breaking changes) over the main one. vnext publishing waits for the dev repo from OvertureMaps/ops-team#299. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- docs/versioning.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/versioning.md b/docs/versioning.md index a0ec43b59..e35479bdd 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -32,9 +32,9 @@ Every distributable package under `packages/*` carries its own independent | Event | Version | Destination | |-------|---------|-------------| -| Push to `vnext` | `.postN+vnext.` | CodeArtifact | | Push to `main`, no bump | `.postN+main.` | CodeArtifact | | Version bump on `main` | `` | GitHub Release, then public PyPI | +| Push to `vnext` | `.postN+vnext.` | Blocked on a dedicated dev repository ([ops-team#299](https://github.com/OvertureMaps/ops-team/issues/299)) | Internal builds use PEP 440 post-releases so they order after the released ``: a consumer pinning `>=1.2.3` resolves `1.2.3.post4+main.abc1234` @@ -42,9 +42,12 @@ from CodeArtifact when present (verified with uv), while `==1.2.3` still selects the clean release. Pin with `>=`, never `>`: PEP 440 excludes post-releases from exclusive ordered comparisons, so `>1.2.3` matches no internal build. `N` is the workflow run number. The -`+main`/`+vnext` local label names the build stream; local labels are ignored -in version comparison and rejected by public PyPI, which keeps these builds -internal by construction. +`+main`/`+vnext` local label names the build stream but does not participate +in version ordering, so the two streams must never share a repository: in a +shared repo, a `>=` consumer can resolve a `vnext` build (breaking changes) +over the `main` one. `vnext` builds publish only once a separate dev +repository exists. Local labels are rejected by public PyPI, which keeps +internal builds off the public index by construction. ### Dependency materialization From 38ac277cfb6b1f5fbd9ce204af955d47af78f1f3 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 16:12:03 -0400 Subject: [PATCH 21/41] [DOCS](versioning) Note the static dual-declaration alternative is not used Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- docs/versioning.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/versioning.md b/docs/versioning.md index e35479bdd..378e41189 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -59,6 +59,13 @@ to rewrite each bare workspace dependency to a floor from the in-repo version (e.g. `overture-schema-common` becomes `overture-schema-common>=0.1.1`). Floors carry the released version only, never a `.postN` suffix. +uv's documented alternative is +[static dual declaration](https://docs.astral.sh/uv/concepts/projects/dependencies/#workspace-member): +hand-maintained specifiers in `project.dependencies` alongside the workspace +source. Not used here; with 13 interdependent packages, every bump PR would +have to touch each dependent's floor by hand. The script derives the same +specifiers at publish time instead. + ### Tag scheme Each package has its own release series: tag `-v..`, From 9486930784a2cd39da792f2746186cab4c3471ff Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 16:33:53 -0400 Subject: [PATCH 22/41] [FEATURE](ci) Enforce major-bump cascade across workspace dependents Dependency floors are materialized from in-repo versions at publish time, so a major bump of a dependency becomes a breaking constraint in every dependent's next publish. check_major_cascade fails the version diff when a package's direct workspace dependency takes a major bump without the package bumping too; direct-dep checking cascades through longer chains since each unbumped link fails its own check. Runs in the PR-time version check (blocks merge) and again in release-trigger before any release is cut. Flagged by @vcschapp in the release-scenarios review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../workflows/scripts/detect_version_bumps.py | 18 ++++- .github/workflows/scripts/package_versions.py | 79 ++++++++++++++++--- docs/versioning.md | 4 + 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/workflows/scripts/detect_version_bumps.py index ecb923cd2..65323283c 100644 --- a/.github/workflows/scripts/detect_version_bumps.py +++ b/.github/workflows/scripts/detect_version_bumps.py @@ -20,7 +20,8 @@ Exit status: 0 Success (including the no-bump case). - 1 A package's version went backwards; that must never land on main. + 1 A package's version went backwards, or a major bump does not cascade + to its dependents; neither must ever land on main. """ from pathlib import Path @@ -30,6 +31,8 @@ import sys import tomllib +from package_versions import check_major_cascade, package_manifests + def semver(pyproject_blob: bytes) -> tuple[int, int, int]: """Parse `project.version` from pyproject.toml bytes into (major, minor, patch).""" @@ -92,6 +95,19 @@ def main() -> None: ) sys.exit(1) + # Belt-and-braces re-check of the major-bump cascade (primary enforcement + # is the PR-time version check). Publishing releases with a non-cascaded + # major bump would poison dependents' materialized floors. + after_manifests = { + path.parent.name: tomllib.loads(path.read_text(encoding="utf-8")) + for path in sorted(Path("packages").glob("*/pyproject.toml")) + } + violations = check_major_cascade(package_manifests(before), after_manifests) + if violations: + for v in violations: + print(f"::error::{v}") + sys.exit(1) + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: out.write(f"count={len(bumps)}\n") out.write(f"bumps={json.dumps(bumps)}\n") diff --git a/.github/workflows/scripts/package_versions.py b/.github/workflows/scripts/package_versions.py index d4865a5c1..3b236f541 100644 --- a/.github/workflows/scripts/package_versions.py +++ b/.github/workflows/scripts/package_versions.py @@ -13,6 +13,10 @@ changed dependencies come first. The dependency order is derived from each package's declared `project.dependencies`, restricted to workspace members. +Also enforces the major-bump cascade: a package whose direct workspace +dependency takes a major bump must take one itself (see +`check_major_cascade`). + Form of the JSON array: [ {"package": "p1", "before": "v1", "after": "v2"}, ... ] @@ -22,7 +26,7 @@ Exit status: 0 Success. - 1 Usage error. + 1 Usage error, or a major bump that does not cascade to its dependents. """ from graphlib import TopologicalSorter @@ -63,6 +67,12 @@ def package_manifests(commit: str) -> dict[str, dict]: def topo_order(manifests: dict[str, dict]) -> list[str]: """Package names sorted so dependencies come before their dependents.""" + graph = dependency_graph(manifests) + return list(TopologicalSorter(graph).static_order()) + + +def dependency_graph(manifests: dict[str, dict]) -> dict[str, set[str]]: + """Map each package directory name to its direct workspace dependencies.""" dist_to_dir = { str(m["project"]["name"]): package for package, m in manifests.items() } @@ -74,21 +84,61 @@ def topo_order(manifests: dict[str, dict]) -> list[str]: if match and match.group(1) in dist_to_dir: deps.add(dist_to_dir[match.group(1)]) graph[package] = deps - return list(TopologicalSorter(graph).static_order()) + return graph + + +def manifest_version(manifests: dict[str, dict], package: str) -> str | None: + """Static `project.version` of `package`, or None if absent/dynamic.""" + manifest = manifests.get(package) + if manifest is None: + return None + value = manifest["project"].get("version") + return str(value) if value is not None else None + + +def check_major_cascade( + before_manifests: dict[str, dict], after_manifests: dict[str, dict] +) -> list[str]: + """ + Enforce that major bumps cascade up the dependency tree. + + Workspace dependency floors are materialized from in-repo versions at + publish time, so a major bump of a dependency silently becomes a breaking + constraint change in every dependent's next publish. A package whose + direct workspace dependency takes a major bump must therefore take a + major bump in the same change. Checking direct dependencies is enough: + each unbumped link in a longer chain fails its own check. + + Returns a list of violation descriptions (empty when compliant). + """ + + def major(version: str | None) -> int | None: + return int(version.split(".")[0]) if version else None + + errors = [] + for package, deps in dependency_graph(after_manifests).items(): + pkg_before = major(manifest_version(before_manifests, package)) + pkg_after = major(manifest_version(after_manifests, package)) + pkg_bumped = pkg_before is not None and pkg_after is not None and pkg_after > pkg_before + + for dep in sorted(deps): + dep_before = major(manifest_version(before_manifests, dep)) + dep_after = major(manifest_version(after_manifests, dep)) + if dep_before is None or dep_after is None or dep_after <= dep_before: + continue + if not pkg_bumped: + errors.append( + f"{package} depends on {dep}, which takes a major bump " + f"({dep_before}.x -> {dep_after}.x), but {package} does not. " + "Major bumps must cascade to dependents." + ) + return errors def diff(before: str, after: str) -> None: before_manifests = package_manifests(before) after_manifests = package_manifests(after) - def version(manifests: dict[str, dict], package: str) -> str | None: - manifest = manifests.get(package) - if manifest is None: - return None - # A dynamic version (no static `project.version`) also maps to None. - value = manifest["project"].get("version") - return str(value) if value is not None else None - # Order from the after commit, which knows about newly added packages; # packages that only exist in before (deleted) are appended at the end. order = topo_order(after_manifests) @@ -97,10 +147,17 @@ def version(manifests: dict[str, dict], package: str) -> str | None: changed = [ {"package": p, "before": b, "after": a} for p in order - if (b := version(before_manifests, p)) != (a := version(after_manifests, p)) + if (b := manifest_version(before_manifests, p)) + != (a := manifest_version(after_manifests, p)) ] print(json.dumps(changed, indent=2)) + violations = check_major_cascade(before_manifests, after_manifests) + if violations: + for v in violations: + print(f"::error::{v}", file=sys.stderr) + sys.exit(1) + if __name__ == "__main__": if len(sys.argv) != 4 or sys.argv[1] != "diff": diff --git a/docs/versioning.md b/docs/versioning.md index 378e41189..1a071e087 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -82,6 +82,10 @@ deliberate, one-time discontinuity. the `Changelog fragment verification` check. - `release-trigger` fails if the target tag already exists, or if a version goes backwards. +- Major bumps cascade: a package whose workspace dependency takes a major bump + must take one itself, since its next publish materializes that dependency as + a breaking `>=` floor. Enforced at PR time by the version check and again by + `release-trigger`. ## How to From dfb4bf891728c46a1e8a6dab0632c27a6588fd45 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 28 Jul 2026 16:41:32 -0400 Subject: [PATCH 23/41] [FEATURE](ci) Sequence internal builds per version from CodeArtifact Replaces run-number N with a per-version sequence: the first internal build of a version is .post0, communicating identical contents to the release, and later builds increment from the highest .postN already published. Costs one CodeArtifact resolve per package, restoring the index_url input and the dry-run workflow's AWS credential steps. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/actions/compute-version/action.yml | 48 +++++++++++++++++-- .../workflows/compute-versions-dry-run.yaml | 26 +++++++--- docs/versioning.md | 4 +- 3 files changed, 66 insertions(+), 12 deletions(-) diff --git a/.github/actions/compute-version/action.yml b/.github/actions/compute-version/action.yml index 475ffc902..7bc288013 100644 --- a/.github/actions/compute-version/action.yml +++ b/.github/actions/compute-version/action.yml @@ -8,12 +8,17 @@ description: > between releases, using a PEP 440 post-release so they order AFTER the released version and are picked up by `>=` specifiers: - - `main`: `.post+main.` - - `vnext`: `.post+vnext.` + - `main`: `.post+main.` + - `vnext`: `.post+vnext.` + + `N` is a per-version sequence: the first internal build of a version is + `.post0` (communicating "identical to the release"), and each subsequent + build increments from the highest `.postN` already published to + CodeArtifact. The `+main`/`+vnext` local label distinguishes the two build streams; local labels are ignored during version comparison so ordering comes from - `.post` alone. Local versions are rejected by public PyPI, which + `.post` alone. Local versions are rejected by public PyPI, which guarantees these builds stay internal. Prerequisites: repo must be checked out and `uv` must be available. @@ -27,6 +32,12 @@ inputs: Branch context naming the build stream. Supported values: `main`, `vnext`. required: true + index_url: + description: > + PyPI simple index URL with embedded credentials for querying + CodeArtifact. Obtain via the `.github/actions/code-artifact` action's + `index_url` output after configuring AWS credentials. + required: true outputs: version: @@ -42,7 +53,7 @@ runs: env: PACKAGE: ${{ inputs.package }} CONTEXT: ${{ inputs.context }} - RUN_NUMBER: ${{ github.run_number }} + INDEX_URL: ${{ inputs.index_url }} SHA: ${{ github.sha }} run: | set -euo pipefail @@ -58,7 +69,34 @@ runs: SEED=$(cd "packages/${PACKAGE}" && uv version --short) SHORT_SHA=$(echo "$SHA" | cut -c1-7) - VERSION="${SEED}.post${RUN_NUMBER}+${CONTEXT}.${SHORT_SHA}" + # Resolve the highest published build of the seed version. The `.*` + # prefix match includes the release itself and its post-releases. + # uv pip compile exits non-zero both when nothing matches (a normal + # "not published yet" result) and on real failures (network/auth/etc). + # Only treat the former as benign; anything else must surface and fail. + LATEST="" + if ! OUTPUT=$(echo "${PACKAGE}==${SEED}.*" \ + | uv pip compile - --index-url "$INDEX_URL" --no-deps --quiet 2>&1); then + if ! echo "$OUTPUT" | grep -qiE 'no solution found|could not find a version|not found in the package registry'; then + echo "ERROR: uv pip compile failed for '${PACKAGE}==${SEED}.*':" >&2 + echo "$OUTPUT" >&2 + exit 1 + fi + else + LATEST=$(echo "$OUTPUT" | grep -oE "==[0-9][^ ]*" | head -1 | cut -c3-) + fi + + # First build of this version is .post0; otherwise increment the + # highest published .postN. + if [[ "$LATEST" =~ \.post([0-9]+) ]]; then + N=$(( ${BASH_REMATCH[1]} + 1 )) + echo "Latest published build: ${LATEST} -> next N: ${N}" + else + N=0 + echo "No published .post build for ${SEED} (latest: '${LATEST:-none}') -> N: 0" + fi + + VERSION="${SEED}.post${N}+${CONTEXT}.${SHORT_SHA}" echo "Computed version for ${PACKAGE} (${CONTEXT}): ${VERSION}" echo "version=${VERSION}" >> "$GITHUB_OUTPUT" \ No newline at end of file diff --git a/.github/workflows/compute-versions-dry-run.yaml b/.github/workflows/compute-versions-dry-run.yaml index fcd8aa716..9d50b15f8 100644 --- a/.github/workflows/compute-versions-dry-run.yaml +++ b/.github/workflows/compute-versions-dry-run.yaml @@ -2,8 +2,8 @@ name: Compute versions (dry run) # Runs on pushes to vnext and main. Computes and logs the version that would # be published for each affected package — but does not build or publish. -# Also runs (read-only) on PRs that touch the compute-version composite -# action or this workflow itself, as a smoke test for those. +# Also runs (read-only) on PRs that touch the compute-version/code-artifact +# composite actions or this workflow itself, as a smoke test for those. # Remove this workflow once Phase 3 publish workflows are live (see # https://github.com/OvertureMaps/schema/issues/509). @@ -14,12 +14,13 @@ on: branches: [main, vnext] paths: - '**/pyproject.toml' - # Test usage: smoke-tests the compute-version composite action (and this - # workflow) against a placeholder context on PRs that touch them, so wiring - # regressions surface before merge. + # Test usage: smoke-tests the compute-version/code-artifact composite + # actions (and this workflow) against a placeholder context on PRs that + # touch them, so wiring regressions surface before merge. pull_request: paths: - '.github/actions/compute-version/**' + - '.github/actions/code-artifact/**' - '.github/workflows/compute-versions-dry-run.yaml' workflow_dispatch: inputs: @@ -75,9 +76,10 @@ jobs: compute-versions: name: Compute version (${{ matrix.package }})${{ github.event_name == 'pull_request' && ' (test)' || '' }} needs: discover - runs-on: ubuntu-slim + runs-on: ubuntu-latest permissions: contents: read + id-token: write # Required for OIDC authentication to AWS strategy: fail-fast: false matrix: @@ -94,6 +96,17 @@ jobs: with: persist-credentials: false + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::505071440022:role/GithubActions_Schema_CodeArtifact_ReadOnly + role-session-name: GitHubActions_${{github.job}}_${{github.run_id}} + + - name: Get CodeArtifact credentials + id: ca + uses: ./.github/actions/code-artifact + # Delegates to the same composite action Phase 3 publish workflows will # use, so the version formula only has one implementation to keep correct. - name: Compute version @@ -102,6 +115,7 @@ jobs: with: package: ${{ matrix.package }} context: ${{ needs.discover.outputs.context }} + index_url: ${{ steps.ca.outputs.index_url }} - name: Report computed version env: diff --git a/docs/versioning.md b/docs/versioning.md index 1a071e087..c08a835c6 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -41,7 +41,9 @@ Internal builds use PEP 440 post-releases so they order after the released from CodeArtifact when present (verified with uv), while `==1.2.3` still selects the clean release. Pin with `>=`, never `>`: PEP 440 excludes post-releases from exclusive ordered comparisons, so `>1.2.3` matches no -internal build. `N` is the workflow run number. The +internal build. `N` is a per-version sequence: the first internal build of a +version is `.post0` (identical contents to the release), incrementing from +the highest `.postN` already published. The `+main`/`+vnext` local label names the build stream but does not participate in version ordering, so the two streams must never share a repository: in a shared repo, a `>=` consumer can resolve a `vnext` build (breaking changes) From 7842f7c0955e0ecaee2e9dbcef48c192a967c636 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 29 Jul 2026 10:17:32 -0400 Subject: [PATCH 24/41] [REFACTOR](deps) Declare static workspace dependency floors, drop materialization Consensus from PR review: maintainers keep explicit >= floors in project.dependencies by hand, uv's documented dual-declaration pattern, instead of a CI step rewriting bare names at publish time. Wheel metadata now carries the floors with no pre-build mutation, and the versioning behavior is visible in the pyproject.toml a contributor actually reads. materialize_workspace_deps.py is deleted; the major-bump cascade check stays as the sanity net. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../workflows/scripts/detect_version_bumps.py | 2 +- .../scripts/materialize_workspace_deps.py | 89 ------------------- .github/workflows/scripts/package_versions.py | 6 +- docs/versioning.md | 33 ++++--- .../changelog.d/557.misc.md | 1 + .../pyproject.toml | 4 +- .../changelog.d/557.misc.md | 1 + packages/overture-schema-annex/pyproject.toml | 2 +- .../changelog.d/557.misc.md | 1 + .../overture-schema-base-theme/pyproject.toml | 4 +- .../changelog.d/557.misc.md | 1 + .../pyproject.toml | 4 +- .../changelog.d/557.misc.md | 1 + packages/overture-schema-cli/pyproject.toml | 4 +- .../changelog.d/557.misc.md | 1 + .../overture-schema-codegen/pyproject.toml | 6 +- .../changelog.d/557.misc.md | 1 + .../overture-schema-common/pyproject.toml | 2 +- .../changelog.d/557.misc.md | 1 + .../pyproject.toml | 4 +- .../changelog.d/557.misc.md | 1 + .../pyproject.toml | 4 +- .../changelog.d/557.misc.md | 1 + .../overture-schema-pyspark/pyproject.toml | 2 +- .../changelog.d/557.misc.md | 1 + .../pyproject.toml | 4 +- .../overture-schema/changelog.d/557.misc.md | 1 + packages/overture-schema/pyproject.toml | 16 ++-- 28 files changed, 59 insertions(+), 139 deletions(-) delete mode 100644 .github/workflows/scripts/materialize_workspace_deps.py create mode 100644 packages/overture-schema-addresses-theme/changelog.d/557.misc.md create mode 100644 packages/overture-schema-annex/changelog.d/557.misc.md create mode 100644 packages/overture-schema-base-theme/changelog.d/557.misc.md create mode 100644 packages/overture-schema-buildings-theme/changelog.d/557.misc.md create mode 100644 packages/overture-schema-cli/changelog.d/557.misc.md create mode 100644 packages/overture-schema-codegen/changelog.d/557.misc.md create mode 100644 packages/overture-schema-common/changelog.d/557.misc.md create mode 100644 packages/overture-schema-divisions-theme/changelog.d/557.misc.md create mode 100644 packages/overture-schema-places-theme/changelog.d/557.misc.md create mode 100644 packages/overture-schema-pyspark/changelog.d/557.misc.md create mode 100644 packages/overture-schema-transportation-theme/changelog.d/557.misc.md diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/workflows/scripts/detect_version_bumps.py index 65323283c..c6dc5e046 100644 --- a/.github/workflows/scripts/detect_version_bumps.py +++ b/.github/workflows/scripts/detect_version_bumps.py @@ -97,7 +97,7 @@ def main() -> None: # Belt-and-braces re-check of the major-bump cascade (primary enforcement # is the PR-time version check). Publishing releases with a non-cascaded - # major bump would poison dependents' materialized floors. + # major bump would poison dependents' declared floors. after_manifests = { path.parent.name: tomllib.loads(path.read_text(encoding="utf-8")) for path in sorted(Path("packages").glob("*/pyproject.toml")) diff --git a/.github/workflows/scripts/materialize_workspace_deps.py b/.github/workflows/scripts/materialize_workspace_deps.py deleted file mode 100644 index f4459ce86..000000000 --- a/.github/workflows/scripts/materialize_workspace_deps.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python3 -# /// script -# requires-python = ">=3.11" -# dependencies = [ -# "tomlkit>=0.13", -# ] -# /// - -""" -Materialize workspace dependency versions into a package's pyproject.toml. - -Run from the repository root, before building a package for publishing: - - uv run ./.github/workflows/scripts/materialize_workspace_deps.py - -Intra-repo dependencies are declared as bare names (e.g. "overture-schema-common") -and resolved at dev time through `[tool.uv.sources]` workspace entries. Built -distributions carry the declared metadata as-is, so without this step a published -wheel would depend on an unconstrained package name. uv has no first-class -feature for this (workspace sources are dev-only and dropped at build time). - -For each dependency that names a workspace member, this script rewrites the -bare name to a floor constraint from the version currently in the repo, e.g. -"overture-schema-common" becomes "overture-schema-common>=0.1.1". The floor is -the released version only; internal `.postN` build suffixes are never written -into dependency constraints. Edits preserve pyproject.toml formatting. - -Dependencies that already carry a constraint, and dependencies on packages -outside this repo, are left untouched. - -Exit status: - 0 Success (including nothing-to-do). - 1 Unknown package, or a workspace dependency's pyproject cannot be read. -""" - -from pathlib import Path -import re -import sys - -import tomlkit - -PACKAGES_DIR = Path("packages") - -# A bare PEP 508 name: no extras, no specifier, no markers. -BARE_NAME = re.compile(r"^[A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?$") - - -def workspace_versions() -> dict[str, str]: - """Map every workspace member's distribution name to its in-repo version.""" - versions: dict[str, str] = {} - for path in sorted(PACKAGES_DIR.glob("*/pyproject.toml")): - project = tomlkit.parse(path.read_text(encoding="utf-8"))["project"] - versions[str(project["name"])] = str(project["version"]) - return versions - - -def materialize(package: str) -> int: - pyproject_path = PACKAGES_DIR / package / "pyproject.toml" - if not pyproject_path.is_file(): - print(f"::error::No such package: {package} ({pyproject_path} missing).") - return 1 - - versions = workspace_versions() - doc = tomlkit.parse(pyproject_path.read_text(encoding="utf-8")) - dependencies = doc["project"].get("dependencies", []) - - changed = 0 - for i, dep in enumerate(dependencies): - name = str(dep).strip() - if not BARE_NAME.match(name) or name not in versions: - continue - floor = f"{name}>={versions[name]}" - dependencies[i] = floor - changed += 1 - print(f"{package}: {name} -> {floor}") - - if changed: - pyproject_path.write_text(tomlkit.dumps(doc), encoding="utf-8") - print(f"{package}: materialized {changed} workspace dependenc{'y' if changed == 1 else 'ies'}.") - else: - print(f"{package}: no bare workspace dependencies to materialize.") - return 0 - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("usage: materialize_workspace_deps.py ", file=sys.stderr) - sys.exit(1) - sys.exit(materialize(sys.argv[1])) diff --git a/.github/workflows/scripts/package_versions.py b/.github/workflows/scripts/package_versions.py index 3b236f541..e9bf67040 100644 --- a/.github/workflows/scripts/package_versions.py +++ b/.github/workflows/scripts/package_versions.py @@ -102,9 +102,9 @@ def check_major_cascade( """ Enforce that major bumps cascade up the dependency tree. - Workspace dependency floors are materialized from in-repo versions at - publish time, so a major bump of a dependency silently becomes a breaking - constraint change in every dependent's next publish. A package whose + Workspace dependency floors are declared statically in each package's + `project.dependencies`, so a major bump of a dependency is a breaking + change behind every dependent's existing floor. A package whose direct workspace dependency takes a major bump must therefore take a major bump in the same change. Checking direct dependencies is enough: each unbumped link in a longer chain fails its own check. diff --git a/docs/versioning.md b/docs/versioning.md index c08a835c6..bcf95365f 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -8,7 +8,7 @@ Reference and how-to for package versions and releases. Branch mechanics and the - [Reference](#reference) - [Version scheme](#version-scheme) - [Version → destination](#version--destination) - - [Dependency materialization](#dependency-materialization) + - [Workspace dependency floors](#workspace-dependency-floors) - [Tag scheme](#tag-scheme) - [Guardrails](#guardrails) - [How to](#how-to) @@ -51,22 +51,19 @@ over the `main` one. `vnext` builds publish only once a separate dev repository exists. Local labels are rejected by public PyPI, which keeps internal builds off the public index by construction. -### Dependency materialization +### Workspace dependency floors -Intra-repo dependencies are declared as bare names and resolved at dev time -via `[tool.uv.sources]` workspace entries, which uv drops at build time. -Before publishing, CI runs -[`materialize_workspace_deps.py`](../.github/workflows/scripts/materialize_workspace_deps.py) -to rewrite each bare workspace dependency to a floor from the in-repo version -(e.g. `overture-schema-common` becomes `overture-schema-common>=0.1.1`). -Floors carry the released version only, never a `.postN` suffix. +Intra-repo dependencies follow uv's +[dual declaration](https://docs.astral.sh/uv/concepts/projects/dependencies/#workspace-member) +pattern: an explicit specifier in `project.dependencies` (e.g. +`overture-schema-common>=0.1.1`) alongside a `[tool.uv.sources]` workspace +entry. Development resolves against the workspace source; built wheels carry +the specifier. -uv's documented alternative is -[static dual declaration](https://docs.astral.sh/uv/concepts/projects/dependencies/#workspace-member): -hand-maintained specifiers in `project.dependencies` alongside the workspace -source. Not used here; with 13 interdependent packages, every bump PR would -have to touch each dependent's floor by hand. The script derives the same -specifiers at publish time instead. +Floors are maintained by hand. Raise a floor only when your package needs +something from the newer dependency version; floors carry released versions +only, never a `.postN` suffix. Major bumps are the exception, they must +cascade (see [Guardrails](#guardrails)). ### Tag scheme @@ -85,9 +82,9 @@ deliberate, one-time discontinuity. - `release-trigger` fails if the target tag already exists, or if a version goes backwards. - Major bumps cascade: a package whose workspace dependency takes a major bump - must take one itself, since its next publish materializes that dependency as - a breaking `>=` floor. Enforced at PR time by the version check and again by - `release-trigger`. + must take one itself, and its floor on that dependency must be raised, since + the old floor would admit a breaking version. Enforced at PR time by the + version check and again by `release-trigger`. ## How to diff --git a/packages/overture-schema-addresses-theme/changelog.d/557.misc.md b/packages/overture-schema-addresses-theme/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-addresses-theme/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-addresses-theme/pyproject.toml b/packages/overture-schema-addresses-theme/pyproject.toml index 9efd0ce6e..7cb105b5e 100644 --- a/packages/overture-schema-addresses-theme/pyproject.toml +++ b/packages/overture-schema-addresses-theme/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", ] description = "Overture Maps addresses theme models and structures" diff --git a/packages/overture-schema-annex/changelog.d/557.misc.md b/packages/overture-schema-annex/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-annex/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-annex/pyproject.toml b/packages/overture-schema-annex/pyproject.toml index af7ea058d..a93efd2da 100644 --- a/packages/overture-schema-annex/pyproject.toml +++ b/packages/overture-schema-annex/pyproject.toml @@ -2,7 +2,7 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] -dependencies = ["overture-schema-common", "overture-schema-system", "pydantic>=2.12.0"] +dependencies = ["overture-schema-common>=0.1.1", "overture-schema-system>=0.1.1", "pydantic>=2.12.0"] description = "Add your description here" version = "0.1.1" license = "MIT" diff --git a/packages/overture-schema-base-theme/changelog.d/557.misc.md b/packages/overture-schema-base-theme/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-base-theme/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-base-theme/pyproject.toml b/packages/overture-schema-base-theme/pyproject.toml index cd0644108..38662bafa 100644 --- a/packages/overture-schema-base-theme/pyproject.toml +++ b/packages/overture-schema-base-theme/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", ] description = "Overture Maps base theme shared structures and models (bathymetry, infrastructure, land, land_cover, land_use, water)" diff --git a/packages/overture-schema-buildings-theme/changelog.d/557.misc.md b/packages/overture-schema-buildings-theme/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-buildings-theme/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-buildings-theme/pyproject.toml b/packages/overture-schema-buildings-theme/pyproject.toml index 2ab6a3352..0cbb9f172 100644 --- a/packages/overture-schema-buildings-theme/pyproject.toml +++ b/packages/overture-schema-buildings-theme/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", ] description = "Overture Maps buildings theme shared structures, building types, and building part types" diff --git a/packages/overture-schema-cli/changelog.d/557.misc.md b/packages/overture-schema-cli/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-cli/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-cli/pyproject.toml b/packages/overture-schema-cli/pyproject.toml index f32fd78ef..45b16434f 100644 --- a/packages/overture-schema-cli/pyproject.toml +++ b/packages/overture-schema-cli/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", "pyyaml>=6.0.2", "click>=8.1", diff --git a/packages/overture-schema-codegen/changelog.d/557.misc.md b/packages/overture-schema-codegen/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-codegen/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-codegen/pyproject.toml b/packages/overture-schema-codegen/pyproject.toml index 60c835553..dc3c09445 100644 --- a/packages/overture-schema-codegen/pyproject.toml +++ b/packages/overture-schema-codegen/pyproject.toml @@ -6,9 +6,9 @@ requires = ["hatchling"] dependencies = [ "click>=8.1", "jinja2>=3.0", - "overture-schema-cli", - "overture-schema-common", - "overture-schema-system", + "overture-schema-cli>=0.1.1", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "tomli>=2.0; python_version < '3.11'", "typing-extensions>=4.0", ] diff --git a/packages/overture-schema-common/changelog.d/557.misc.md b/packages/overture-schema-common/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-common/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-common/pyproject.toml b/packages/overture-schema-common/pyproject.toml index bf80b50ac..e63298425 100644 --- a/packages/overture-schema-common/pyproject.toml +++ b/packages/overture-schema-common/pyproject.toml @@ -12,7 +12,7 @@ description = "Common components that are shared across Overture theme schemas" requires-python = ">=3.10" license = "MIT" dependencies = [ - "overture-schema-system", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", "shapely>=2.1.1", ] diff --git a/packages/overture-schema-divisions-theme/changelog.d/557.misc.md b/packages/overture-schema-divisions-theme/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-divisions-theme/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-divisions-theme/pyproject.toml b/packages/overture-schema-divisions-theme/pyproject.toml index c361c2783..e1ac36d2e 100644 --- a/packages/overture-schema-divisions-theme/pyproject.toml +++ b/packages/overture-schema-divisions-theme/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ { name = "Overture Maps Schema Working Group" }, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", ] description = "Overture Maps divisions theme shared structures, division, division area and division boundary types" diff --git a/packages/overture-schema-places-theme/changelog.d/557.misc.md b/packages/overture-schema-places-theme/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-places-theme/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-places-theme/pyproject.toml b/packages/overture-schema-places-theme/pyproject.toml index 995fd707a..811a3d92a 100644 --- a/packages/overture-schema-places-theme/pyproject.toml +++ b/packages/overture-schema-places-theme/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic[email]>=2.12.0", ] description = "Overture Maps places theme with place type models" diff --git a/packages/overture-schema-pyspark/changelog.d/557.misc.md b/packages/overture-schema-pyspark/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-pyspark/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-pyspark/pyproject.toml b/packages/overture-schema-pyspark/pyproject.toml index 646dc88c4..027a5ae60 100644 --- a/packages/overture-schema-pyspark/pyproject.toml +++ b/packages/overture-schema-pyspark/pyproject.toml @@ -5,7 +5,7 @@ requires = ["hatchling"] [project] dependencies = [ "click>=8.0", - "overture-schema-system", + "overture-schema-system>=0.1.1", "pyspark>=3.4", ] description = "PySpark validation expressions for Overture Maps data" diff --git a/packages/overture-schema-transportation-theme/changelog.d/557.misc.md b/packages/overture-schema-transportation-theme/changelog.d/557.misc.md new file mode 100644 index 000000000..67efb8d14 --- /dev/null +++ b/packages/overture-schema-transportation-theme/changelog.d/557.misc.md @@ -0,0 +1 @@ +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema-transportation-theme/pyproject.toml b/packages/overture-schema-transportation-theme/pyproject.toml index ee241cf6d..5fa8df617 100644 --- a/packages/overture-schema-transportation-theme/pyproject.toml +++ b/packages/overture-schema-transportation-theme/pyproject.toml @@ -3,8 +3,8 @@ maintainers = [ { name = "Overture Maps Schema Working Group" }, ] dependencies = [ - "overture-schema-common", - "overture-schema-system", + "overture-schema-common>=0.1.1", + "overture-schema-system>=0.1.1", "pydantic>=2.12.0", ] description = "Overture Maps transportation theme with shared structures and connector and segment types" diff --git a/packages/overture-schema/changelog.d/557.misc.md b/packages/overture-schema/changelog.d/557.misc.md index 79ecd1dc4..3dd1286bc 100644 --- a/packages/overture-schema/changelog.d/557.misc.md +++ b/packages/overture-schema/changelog.d/557.misc.md @@ -1 +1,2 @@ Added per-package release-trigger workflow, towncrier changelog fragments, and a fragment-required CI check. +Declared explicit version floors for workspace dependencies. diff --git a/packages/overture-schema/pyproject.toml b/packages/overture-schema/pyproject.toml index bb748c46e..e49866c40 100644 --- a/packages/overture-schema/pyproject.toml +++ b/packages/overture-schema/pyproject.toml @@ -3,16 +3,16 @@ maintainers = [ {name = "Overture Maps Schema Working Group"}, ] dependencies = [ - "overture-schema-addresses-theme", - "overture-schema-base-theme", - "overture-schema-buildings-theme", - "overture-schema-divisions-theme", - "overture-schema-places-theme", - "overture-schema-transportation-theme", - "overture-schema-common", + "overture-schema-addresses-theme>=0.1.1", + "overture-schema-base-theme>=0.1.1", + "overture-schema-buildings-theme>=0.1.1", + "overture-schema-divisions-theme>=0.1.1", + "overture-schema-places-theme>=0.1.1", + "overture-schema-transportation-theme>=0.1.1", + "overture-schema-common>=0.1.1", "pydantic>=2.12.0", "pyyaml>=6.0.2", - "overture-schema-cli", + "overture-schema-cli>=0.1.1", ] description = "Complete Overture Maps schema collection with all themes and types" version = "1.17.1" From 93ead4e16fd12320e947ac1bb27cdc9a520a359a Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 29 Jul 2026 10:20:01 -0400 Subject: [PATCH 25/41] [BUG](ci) Sync the publish workflow from the committed lockfile Bare uv sync could resolve newer dependency versions than the lock the change was tested against. --locked installs exactly uv.lock and fails if it is stale, matching check-python-code. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/publish-python-packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-python-packages.yaml b/.github/workflows/publish-python-packages.yaml index 6fa5c5599..57e29ab8f 100644 --- a/.github/workflows/publish-python-packages.yaml +++ b/.github/workflows/publish-python-packages.yaml @@ -66,7 +66,7 @@ jobs: persist-credentials: false - name: Sync code to make packages visible to Python - run: uv sync --all-packages + run: uv sync --locked --all-packages - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@517a711dbcd0e402f90c77e7e2f81e849156e31d # v6.2.2 From 40d56eb5ac393690bbb8ed7933d07b2e5af18830 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 29 Jul 2026 15:43:41 -0400 Subject: [PATCH 26/41] [DOCS](changelog) Add copy-paste quick start for changelog fragments New contributors hit the fragment check before they ever read versioning.md, so give them a 30-second copy-paste answer: quick-start section up top of the how-to, the CI failure message pointing at it, and the changelog.d keeper READMEs linking it. The old failure message also told contributors to run towncrier build, which is the release-time maintainer step, not theirs. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../workflows/require-changelog-fragment.yaml | 7 +++--- CONTRIBUTING.md | 5 ++++- docs/versioning.md | 22 +++++++++++++++++++ .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../overture-schema-cli/changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../changelog.d/README.md | 2 +- .../overture-schema/changelog.d/README.md | 2 +- 16 files changed, 42 insertions(+), 18 deletions(-) diff --git a/.github/workflows/require-changelog-fragment.yaml b/.github/workflows/require-changelog-fragment.yaml index 197550e86..861d6e021 100644 --- a/.github/workflows/require-changelog-fragment.yaml +++ b/.github/workflows/require-changelog-fragment.yaml @@ -69,10 +69,9 @@ jobs: echo "::error::These packages changed but include no changelog update:" printf '%s' "$missing" echo "" - echo "Add a towncrier fragment at packages//changelog.d/..md" - echo "(type: breaking | feature | bugfix | docs | misc), then run" - echo '`uvx towncrier build --config pyproject.toml --dir packages/`' - echo "to fold it into CHANGELOG.md. See docs/versioning.md." + echo "Fix: one markdown file per package, named packages//changelog.d/..md" + echo "(type: breaking | feature | bugfix | docs | misc), body = one past-tense sentence." + echo "Quick start: docs/versioning.md#changelog-quick-start" exit 1 fi echo "Changelog fragment check passed." \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1d066bd63..b8fd5805c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -129,7 +129,10 @@ merge to `main`; run `git pull --rebase` before pushing again. - Every package versions and releases independently. Consumers pin only `overture-schema`, which pulls in the theme and support packages for a coherent set. -- Any change to a package **requires a changelog fragment**. Add one under +- Any change to a package **requires a changelog fragment**: one sentence in + one file, see the + [changelog quick start](docs/versioning.md#changelog-quick-start). Add one + under `packages//changelog.d/` and run `uvx towncrier build --config pyproject.toml --dir packages/`. CI enforces it. diff --git a/docs/versioning.md b/docs/versioning.md index bcf95365f..faf56f71d 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -12,6 +12,7 @@ Reference and how-to for package versions and releases. Branch mechanics and the - [Tag scheme](#tag-scheme) - [Guardrails](#guardrails) - [How to](#how-to) + - [Changelog quick start](#changelog-quick-start) - [Add a changelog fragment](#add-a-changelog-fragment) - [Cut a release](#cut-a-release) - [Why](#why) @@ -88,6 +89,27 @@ deliberate, one-time discontinuity. ## How to +### Changelog quick start + +The `Changelog fragment verification` check failed, or you're about to open a +PR that touches a package. The whole fix, using a places-theme bugfix as the +example: + +```bash +# 1. One small markdown file, named ..md +echo 'Fixed brand enum values rejecting valid entries.' \ + > packages/overture-schema-places-theme/changelog.d/561.bugfix.md + +# 2. Commit it with your change +git add packages/overture-schema-places-theme/changelog.d/561.bugfix.md +git commit -m 'Add changelog fragment' +``` + +That's the entire contribution-time cost: one sentence in one file, one per +changed package. No tool to install, no config to touch. towncrier only runs +at release time, when a maintainer folds the accumulated fragments into +`CHANGELOG.md` (see [Cut a release](#cut-a-release)). + ### Add a changelog fragment Release notes are assembled from diff --git a/packages/overture-schema-addresses-theme/changelog.d/README.md b/packages/overture-schema-addresses-theme/changelog.d/README.md index 1cdb08bb4..83ffd44b1 100644 --- a/packages/overture-schema-addresses-theme/changelog.d/README.md +++ b/packages/overture-schema-addresses-theme/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-annex/changelog.d/README.md b/packages/overture-schema-annex/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-annex/changelog.d/README.md +++ b/packages/overture-schema-annex/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-base-theme/changelog.d/README.md b/packages/overture-schema-base-theme/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-base-theme/changelog.d/README.md +++ b/packages/overture-schema-base-theme/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-buildings-theme/changelog.d/README.md b/packages/overture-schema-buildings-theme/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-buildings-theme/changelog.d/README.md +++ b/packages/overture-schema-buildings-theme/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-cli/changelog.d/README.md b/packages/overture-schema-cli/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-cli/changelog.d/README.md +++ b/packages/overture-schema-cli/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-codegen/changelog.d/README.md b/packages/overture-schema-codegen/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-codegen/changelog.d/README.md +++ b/packages/overture-schema-codegen/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-common/changelog.d/README.md b/packages/overture-schema-common/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-common/changelog.d/README.md +++ b/packages/overture-schema-common/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-divisions-theme/changelog.d/README.md b/packages/overture-schema-divisions-theme/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-divisions-theme/changelog.d/README.md +++ b/packages/overture-schema-divisions-theme/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-places-theme/changelog.d/README.md b/packages/overture-schema-places-theme/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-places-theme/changelog.d/README.md +++ b/packages/overture-schema-places-theme/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-pyspark/changelog.d/README.md b/packages/overture-schema-pyspark/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-pyspark/changelog.d/README.md +++ b/packages/overture-schema-pyspark/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-system/changelog.d/README.md b/packages/overture-schema-system/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-system/changelog.d/README.md +++ b/packages/overture-schema-system/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema-transportation-theme/changelog.d/README.md b/packages/overture-schema-transportation-theme/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema-transportation-theme/changelog.d/README.md +++ b/packages/overture-schema-transportation-theme/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is diff --git a/packages/overture-schema/changelog.d/README.md b/packages/overture-schema/changelog.d/README.md index 93297ce0d..0dc9b8a80 100644 --- a/packages/overture-schema/changelog.d/README.md +++ b/packages/overture-schema/changelog.d/README.md @@ -10,7 +10,7 @@ changelog.d/..md Types, body format, the preview command, and when a fragment is required are documented once in -[docs/versioning.md -> Add a changelog fragment](../../../docs/versioning.md#add-a-changelog-fragment). +[docs/versioning.md -> Changelog quick start](../../../docs/versioning.md#changelog-quick-start). > [!NOTE] > This README also keeps `changelog.d/` tracked in git, so no `.gitkeep` is From 2fb1432faa25601d0536293bbd558fbb8227214a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Wed, 29 Jul 2026 20:45:39 +0000 Subject: [PATCH 27/41] fix: resolve merge conflict in reusable-check-python-package-versions.yaml --- ...eusable-check-python-package-versions.yaml | 54 ------------------- 1 file changed, 54 deletions(-) diff --git a/.github/workflows/reusable-check-python-package-versions.yaml b/.github/workflows/reusable-check-python-package-versions.yaml index b15d80264..6f2d68258 100644 --- a/.github/workflows/reusable-check-python-package-versions.yaml +++ b/.github/workflows/reusable-check-python-package-versions.yaml @@ -71,63 +71,13 @@ jobs: changed_packages: ${{ steps.save-changes.outputs.changed_packages }} num_changed_packages: ${{ steps.save-changes.outputs.num_changed_packages }} steps: -<<<<<<< HEAD - name: Check out code - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 -======= - - name: Install jq - run: sudo apt-get update && sudo apt-get install -y jq - - - name: Install uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - with: - version: latest - - - name: Check out code before change uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 ->>>>>>> origin/main with: fetch-depth: 0 # Both comparison commits must be reachable persist-credentials: false -<<<<<<< HEAD - name: Diff package versions -======= - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version-file: .python-version - - - name: Sync code before change to make packages visible to Python - run: uv sync --all-packages - - - name: Capture package versions before change - run: uv run python ./.github/workflows/scripts/package-versions.py collect > /tmp/package-versions-before.json - - - name: Check out code after change - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - ref: ${{ inputs.after_commit }} - persist-credentials: false - - - name: Sync code after change to make packages visible to Python - run: uv sync --all-packages --refresh - - - name: Capture package versions after change - run: uv run python ./.github/workflows/scripts/package-versions.py collect > /tmp/package-versions-after.json - - - name: Compare package versions before and after change - run: | - uv run python ./.github/workflows/scripts/package-versions.py compare \ - /tmp/package-versions-before.json \ - /tmp/package-versions-after.json \ - >/tmp/package-version-diff.json - - - name: Print changed versions - run: cat /tmp/package-version-diff.json - - - name: Save changed versions as output ->>>>>>> origin/main id: save-changes env: BEFORE: ${{ inputs.before_commit }} @@ -145,11 +95,7 @@ jobs: - name: Configure AWS credentials if: steps.save-changes.outputs.num_changed_packages > 0 -<<<<<<< HEAD - uses: aws-actions/configure-aws-credentials@e7f100cf4c008499ea8adda475de1042d6975c7b # v6.2.0 -======= uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6.2.3 ->>>>>>> origin/main with: aws-region: ${{ inputs.aws_region }} role-to-assume: arn:aws:iam::${{ inputs.aws_account_id }}:role/${{ inputs.aws_iam_role_name }} From e3b715516ec1514f482898ac75b221a5f28b32ce Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 13:34:27 -0400 Subject: [PATCH 28/41] [DOCS](changelog) Denote text language on keeper README code fences Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- packages/overture-schema-base-theme/changelog.d/README.md | 2 +- packages/overture-schema-buildings-theme/changelog.d/README.md | 2 +- packages/overture-schema-cli/changelog.d/README.md | 2 +- packages/overture-schema-codegen/changelog.d/README.md | 2 +- packages/overture-schema-common/changelog.d/README.md | 2 +- packages/overture-schema-divisions-theme/changelog.d/README.md | 2 +- packages/overture-schema-places-theme/changelog.d/README.md | 2 +- packages/overture-schema-pyspark/changelog.d/README.md | 2 +- packages/overture-schema-system/changelog.d/README.md | 2 +- .../overture-schema-transportation-theme/changelog.d/README.md | 2 +- packages/overture-schema/changelog.d/README.md | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/packages/overture-schema-base-theme/changelog.d/README.md b/packages/overture-schema-base-theme/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-base-theme/changelog.d/README.md +++ b/packages/overture-schema-base-theme/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-buildings-theme/changelog.d/README.md b/packages/overture-schema-buildings-theme/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-buildings-theme/changelog.d/README.md +++ b/packages/overture-schema-buildings-theme/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-cli/changelog.d/README.md b/packages/overture-schema-cli/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-cli/changelog.d/README.md +++ b/packages/overture-schema-cli/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-codegen/changelog.d/README.md b/packages/overture-schema-codegen/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-codegen/changelog.d/README.md +++ b/packages/overture-schema-codegen/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-common/changelog.d/README.md b/packages/overture-schema-common/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-common/changelog.d/README.md +++ b/packages/overture-schema-common/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-divisions-theme/changelog.d/README.md b/packages/overture-schema-divisions-theme/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-divisions-theme/changelog.d/README.md +++ b/packages/overture-schema-divisions-theme/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-places-theme/changelog.d/README.md b/packages/overture-schema-places-theme/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-places-theme/changelog.d/README.md +++ b/packages/overture-schema-places-theme/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-pyspark/changelog.d/README.md b/packages/overture-schema-pyspark/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-pyspark/changelog.d/README.md +++ b/packages/overture-schema-pyspark/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-system/changelog.d/README.md b/packages/overture-schema-system/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-system/changelog.d/README.md +++ b/packages/overture-schema-system/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema-transportation-theme/changelog.d/README.md b/packages/overture-schema-transportation-theme/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema-transportation-theme/changelog.d/README.md +++ b/packages/overture-schema-transportation-theme/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` diff --git a/packages/overture-schema/changelog.d/README.md b/packages/overture-schema/changelog.d/README.md index 0dc9b8a80..83ffd44b1 100644 --- a/packages/overture-schema/changelog.d/README.md +++ b/packages/overture-schema/changelog.d/README.md @@ -4,7 +4,7 @@ One file per change to this package (including patch-level fixes and internal work): -``` +```text changelog.d/..md ``` From 932b5de5598d9ac24e28885487c418afd9f74f35 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 13:38:10 -0400 Subject: [PATCH 29/41] [DOCS](ci) Fix stale major.minor.0 in create-package-release input description Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/actions/create-package-release/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/create-package-release/action.yml b/.github/actions/create-package-release/action.yml index 96ba10965..0f92d1b3c 100644 --- a/.github/actions/create-package-release/action.yml +++ b/.github/actions/create-package-release/action.yml @@ -16,7 +16,7 @@ inputs: description: Package directory name under packages/ (e.g. overture-schema). required: true version: - description: Release version, major.minor.0 (e.g. 1.18.0). + description: Release version, major.minor.patch (e.g. 1.18.0). required: true tag: description: Release tag (e.g. overture-schema-v1.18.0). From a06aa690c0672e1326da3182f82215f187b8d380 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 13:47:41 -0400 Subject: [PATCH 30/41] [REFACTOR](ci) Wrap bump detection in a composite action Converts the release-trigger detect step to a detect-version-bumps composite action with declared before input and count/bumps outputs, matching compute-version and create-package-release. The script now takes the commit as an argument and writes outputs to stdout (progress to stderr); the action owns the GITHUB_OUTPUT redirection. Suggested by @vcschapp in review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../actions/detect-version-bumps/action.yml | 39 +++++++++++++++ .../detect_version_bumps.py | 48 +++++++++++-------- .github/workflows/release-trigger.yaml | 6 +-- 3 files changed, 69 insertions(+), 24 deletions(-) create mode 100644 .github/actions/detect-version-bumps/action.yml rename .github/{workflows/scripts => actions/detect-version-bumps}/detect_version_bumps.py (74%) diff --git a/.github/actions/detect-version-bumps/action.yml b/.github/actions/detect-version-bumps/action.yml new file mode 100644 index 000000000..8f8f67ea4 --- /dev/null +++ b/.github/actions/detect-version-bumps/action.yml @@ -0,0 +1,39 @@ +name: Detect version bumps +description: > + Detects per-package version bumps between two commits. + + Compares each `packages/*/pyproject.toml` at the checked-out tree against + its content at `before`. Any `..` increase is a bump; + version decreases and non-cascaded major bumps fail the action (see + docs/versioning.md). + + Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` + is reachable. + +inputs: + before: + description: The base commit SHA to compare against (e.g. github.event.before). + required: true + +outputs: + count: + description: Number of bumped packages. + value: ${{ steps.detect.outputs.count }} + bumps: + description: > + JSON array of {"package", "version", "tag"} objects, one per bump. + Suitable as a matrix include list. + value: ${{ steps.detect.outputs.bumps }} + +runs: + using: composite + steps: + - name: Detect version bumps + id: detect + shell: bash + env: + BEFORE: ${{ inputs.before }} + # The script imports the shared package_versions module. + PYTHONPATH: .github/workflows/scripts + run: | + python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" "$BEFORE" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/scripts/detect_version_bumps.py b/.github/actions/detect-version-bumps/detect_version_bumps.py similarity index 74% rename from .github/workflows/scripts/detect_version_bumps.py rename to .github/actions/detect-version-bumps/detect_version_bumps.py index c6dc5e046..ce7f34d99 100644 --- a/.github/workflows/scripts/detect_version_bumps.py +++ b/.github/actions/detect-version-bumps/detect_version_bumps.py @@ -3,30 +3,32 @@ """ Detect per-package version bumps between two commits on main. -Run from the repository root by the `Release trigger` workflow. Compares each -`packages/*/pyproject.toml` at the pushed commit (the working tree) against its +Run from the repository root: + + python3 detect_version_bumps.py + +Compares each `packages/*/pyproject.toml` at the working tree against its content at the `before` commit, and records the packages whose `..` increased. All three components are human-owned (see docs/versioning.md); any increase, including patch-only, cuts a release. -Environment: - BEFORE The `before` commit SHA (github.event.before). - GITHUB_OUTPUT Step output file; receives `count` and `bumps`. +Requires the shared `package_versions` module on PYTHONPATH (it lives in +`.github/workflows/scripts/`); the `detect-version-bumps` action wires this +up. -Outputs (written to $GITHUB_OUTPUT): +Output (stdout, `$GITHUB_OUTPUT` format; progress goes to stderr): count= Number of bumped packages. bumps= JSON array of {"package", "version", "tag"} objects, one per bump. Consumed as a matrix by the release job. Exit status: 0 Success (including the no-bump case). - 1 A package's version went backwards, or a major bump does not cascade - to its dependents; neither must ever land on main. + 1 Usage error, a package's version went backwards, or a major bump does + not cascade to its dependents; none of these must ever land on main. """ from pathlib import Path import json -import os import subprocess import sys import tomllib @@ -56,9 +58,11 @@ def git_show(commit: str, path: str) -> bytes | None: return result.stdout if result.returncode == 0 else None -def main() -> None: - before = os.environ["BEFORE"] +def info(message: str) -> None: + print(message, file=sys.stderr) + +def main(before: str) -> None: bumps: list[dict[str, str]] = [] errors: list[str] = [] @@ -70,7 +74,7 @@ def main() -> None: before_blob = git_show(before, pyproject) if before_blob is None: - print(f"No readable {pyproject} at {before}, skipping {package}.") + info(f"No readable {pyproject} at {before}, skipping {package}.") continue current = semver(before_blob) @@ -84,12 +88,12 @@ def main() -> None: errors.append(f"{package}: {before_str} -> {after_str}") continue - print(f"{package}: {before_str} -> {after_str} (bump)") + info(f"{package}: {before_str} -> {after_str} (bump)") bumps.append({"package": package, "version": after_str, "tag": f"{package}-v{after_str}"}) if errors: for e in errors: - print( + info( f"::error::{e}: version went backwards. Version decreases " "must never land on main; revert or fix the version." ) @@ -105,16 +109,18 @@ def main() -> None: violations = check_major_cascade(package_manifests(before), after_manifests) if violations: for v in violations: - print(f"::error::{v}") + info(f"::error::{v}") sys.exit(1) - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as out: - out.write(f"count={len(bumps)}\n") - out.write(f"bumps={json.dumps(bumps)}\n") - if not bumps: - print("No version bumps detected.") + info("No version bumps detected.") + + print(f"count={len(bumps)}") + print(f"bumps={json.dumps(bumps)}") if __name__ == "__main__": - main() + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} BEFORE_COMMIT", file=sys.stderr) + sys.exit(1) + main(sys.argv[1]) diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 162b721f7..0fde5a6b3 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -52,9 +52,9 @@ jobs: - name: Detect version bumps id: detect - env: - BEFORE: ${{ github.event.before }} - run: python3 ./.github/workflows/scripts/detect_version_bumps.py + uses: ./.github/actions/detect-version-bumps + with: + before: ${{ github.event.before }} release: name: Release ${{ matrix.package }} ${{ matrix.version }} From 20bed8d21853cdb252d69557ce456dee1ccdda33 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 13:54:19 -0400 Subject: [PATCH 31/41] [REFACTOR](ci) Reject non-plain versions with a clear error, stdlib only Review suggested packaging.version.parse for PEP 440 variants, but the model only permits plain X.Y.Z in pyproject.toml, so a 1.2.3rc4 should fail loudly rather than parse. Strict regex + explicit ValueError keeps the scripts dependency-free and gives a better message than the int() conversion error. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-version-bumps/detect_version_bumps.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/actions/detect-version-bumps/detect_version_bumps.py b/.github/actions/detect-version-bumps/detect_version_bumps.py index ce7f34d99..94989f975 100644 --- a/.github/actions/detect-version-bumps/detect_version_bumps.py +++ b/.github/actions/detect-version-bumps/detect_version_bumps.py @@ -29,18 +29,29 @@ from pathlib import Path import json +import re import subprocess import sys import tomllib from package_versions import check_major_cascade, package_manifests +# Released versions are plain X.Y.Z by policy (docs/versioning.md); PEP 440 +# variants like 1.2.3rc4 or 1.2.3.post1 must not appear in pyproject.toml. +PLAIN_SEMVER = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") + def semver(pyproject_blob: bytes) -> tuple[int, int, int]: """Parse `project.version` from pyproject.toml bytes into (major, minor, patch).""" version = str(tomllib.loads(pyproject_blob.decode("utf-8"))["project"]["version"]) - major, minor, patch, *_ = version.split(".") - return int(major), int(minor), int(patch) + match = PLAIN_SEMVER.match(version) + if not match: + raise ValueError( + f"version {version!r} is not plain ..; " + "pre-release/post-release segments are not allowed in pyproject.toml " + "(see docs/versioning.md)" + ) + return int(match.group(1)), int(match.group(2)), int(match.group(3)) def git_show(commit: str, path: str) -> bytes | None: From adb4125044b4f36acfac8bef05cddd9830a259b3 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 13:56:40 -0400 Subject: [PATCH 32/41] [REFACTOR](ci) Rename current to before; surface diagnostics on unreadable commits before/after now pair naturally in the bump loop (the commit arg is before_commit). package_manifests emits a ::notice:: when a commit's tree is unreadable and a ::debug:: with the git stderr, instead of silently returning empty. Both suggested by @vcschapp in review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../detect-version-bumps/detect_version_bumps.py | 16 ++++++++-------- .github/workflows/scripts/package_versions.py | 13 +++++++++++-- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/.github/actions/detect-version-bumps/detect_version_bumps.py b/.github/actions/detect-version-bumps/detect_version_bumps.py index 94989f975..aae6986e1 100644 --- a/.github/actions/detect-version-bumps/detect_version_bumps.py +++ b/.github/actions/detect-version-bumps/detect_version_bumps.py @@ -73,7 +73,7 @@ def info(message: str) -> None: print(message, file=sys.stderr) -def main(before: str) -> None: +def main(before_commit: str) -> None: bumps: list[dict[str, str]] = [] errors: list[str] = [] @@ -83,19 +83,19 @@ def main(before: str) -> None: after = semver(path.read_bytes()) - before_blob = git_show(before, pyproject) + before_blob = git_show(before_commit, pyproject) if before_blob is None: - info(f"No readable {pyproject} at {before}, skipping {package}.") + info(f"No readable {pyproject} at {before_commit}, skipping {package}.") continue - current = semver(before_blob) + before = semver(before_blob) - if after == current: + if after == before: continue - before_str = ".".join(map(str, current)) + before_str = ".".join(map(str, before)) after_str = ".".join(map(str, after)) - if after < current: + if after < before: errors.append(f"{package}: {before_str} -> {after_str}") continue @@ -117,7 +117,7 @@ def main(before: str) -> None: path.parent.name: tomllib.loads(path.read_text(encoding="utf-8")) for path in sorted(Path("packages").glob("*/pyproject.toml")) } - violations = check_major_cascade(package_manifests(before), after_manifests) + violations = check_major_cascade(package_manifests(before_commit), after_manifests) if violations: for v in violations: info(f"::error::{v}") diff --git a/.github/workflows/scripts/package_versions.py b/.github/workflows/scripts/package_versions.py index e9bf67040..108b8247b 100644 --- a/.github/workflows/scripts/package_versions.py +++ b/.github/workflows/scripts/package_versions.py @@ -51,8 +51,17 @@ def package_manifests(commit: str) -> dict[str, dict]: """Map package directory name -> parsed pyproject.toml at `commit`.""" try: listing = git("ls-tree", "--name-only", commit, f"{PACKAGES_DIR}/") - except subprocess.CalledProcessError: - return {} # commit unreadable (force-push) or no packages dir yet + except subprocess.CalledProcessError as e: + # Expected when the commit is unreadable (force-push, history rewrite) + # or predates the packages directory; anything else deserves eyes. + print( + f"::notice::No readable {PACKAGES_DIR}/ tree at {commit} " + "(force-push or no packages directory yet); treating as empty.", + file=sys.stderr, + ) + stderr = e.stderr.decode("utf-8", errors="replace").strip() + print(f"::debug::git ls-tree failed: {stderr}", file=sys.stderr) + return {} manifests: dict[str, dict] = {} for line in listing.splitlines(): From 85176cbc2aa1230d5aa2f4c885f1657a7ea31087 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 14:02:47 -0400 Subject: [PATCH 33/41] [REFACTOR](ci) Compose bump detection from the shared version diff detect_version_bumps.py no longer reads git at all: it is a pure filter from the package_versions.py diff JSON (stdin) to releasable bumps (stdout), applying the plain-semver and no-decrease policies. The detect-version-bumps action pipes the two scripts together, so the DRY lives at the action layer; the cross-script import and its PYTHONPATH wiring are gone. Cascade enforcement rides the first stage's exit status through pipefail. Addresses the duplication flagged by @vcschapp in review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../actions/detect-version-bumps/action.yml | 12 +- .../detect_version_bumps.py | 106 ++++++------------ 2 files changed, 39 insertions(+), 79 deletions(-) diff --git a/.github/actions/detect-version-bumps/action.yml b/.github/actions/detect-version-bumps/action.yml index 8f8f67ea4..6e4e65c1f 100644 --- a/.github/actions/detect-version-bumps/action.yml +++ b/.github/actions/detect-version-bumps/action.yml @@ -2,9 +2,10 @@ name: Detect version bumps description: > Detects per-package version bumps between two commits. - Compares each `packages/*/pyproject.toml` at the checked-out tree against - its content at `before`. Any `..` increase is a bump; - version decreases and non-cascaded major bumps fail the action (see + Composes the shared `package_versions.py diff` (reads versions from git, + enforces the major-bump cascade) with `detect_version_bumps.py` (filters + the diff to releasable bumps). Any `..` increase is a + bump; version decreases and non-cascaded major bumps fail the action (see docs/versioning.md). Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` @@ -33,7 +34,6 @@ runs: shell: bash env: BEFORE: ${{ inputs.before }} - # The script imports the shared package_versions module. - PYTHONPATH: .github/workflows/scripts run: | - python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" "$BEFORE" >> "$GITHUB_OUTPUT" + python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" HEAD \ + | python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" >> "$GITHUB_OUTPUT" diff --git a/.github/actions/detect-version-bumps/detect_version_bumps.py b/.github/actions/detect-version-bumps/detect_version_bumps.py index aae6986e1..fabdd9aef 100644 --- a/.github/actions/detect-version-bumps/detect_version_bumps.py +++ b/.github/actions/detect-version-bumps/detect_version_bumps.py @@ -1,106 +1,82 @@ #!/usr/bin/env python3 """ -Detect per-package version bumps between two commits on main. +Filter a package version diff down to releasable bumps. -Run from the repository root: +Reads the JSON array produced by `package_versions.py diff` on stdin: - python3 detect_version_bumps.py + [ {"package": "p1", "before": "v1", "after": "v2"}, ... ] -Compares each `packages/*/pyproject.toml` at the working tree against its -content at the `before` commit, and records the packages whose -`..` increased. All three components are human-owned -(see docs/versioning.md); any increase, including patch-only, cuts a release. +and emits `$GITHUB_OUTPUT` lines on stdout (progress goes to stderr): -Requires the shared `package_versions` module on PYTHONPATH (it lives in -`.github/workflows/scripts/`); the `detect-version-bumps` action wires this -up. - -Output (stdout, `$GITHUB_OUTPUT` format; progress goes to stderr): count= Number of bumped packages. bumps= JSON array of {"package", "version", "tag"} objects, one per bump. Consumed as a matrix by the release job. +Policy applied (see docs/versioning.md): + - Released versions must be plain `..`; PEP 440 + variants like `1.2.3rc4` fail loudly. + - A version decrease fails: it must never land on main. + - Added packages (`before` null) and removed packages (`after` null) are + not releases; they are skipped. + +The `detect-version-bumps` action composes this with `package_versions.py`, +which owns reading versions from git (and enforces the major-bump cascade +via its own exit status). + Exit status: 0 Success (including the no-bump case). - 1 Usage error, a package's version went backwards, or a major bump does - not cascade to its dependents; none of these must ever land on main. + 1 A version is not plain X.Y.Z, or went backwards. """ -from pathlib import Path import json import re -import subprocess import sys -import tomllib - -from package_versions import check_major_cascade, package_manifests # Released versions are plain X.Y.Z by policy (docs/versioning.md); PEP 440 # variants like 1.2.3rc4 or 1.2.3.post1 must not appear in pyproject.toml. PLAIN_SEMVER = re.compile(r"^(\d+)\.(\d+)\.(\d+)$") -def semver(pyproject_blob: bytes) -> tuple[int, int, int]: - """Parse `project.version` from pyproject.toml bytes into (major, minor, patch).""" - version = str(tomllib.loads(pyproject_blob.decode("utf-8"))["project"]["version"]) +def semver(package: str, version: str) -> tuple[int, int, int]: match = PLAIN_SEMVER.match(version) if not match: raise ValueError( - f"version {version!r} is not plain ..; " + f"{package}: version {version!r} is not plain ..; " "pre-release/post-release segments are not allowed in pyproject.toml " "(see docs/versioning.md)" ) return int(match.group(1)), int(match.group(2)), int(match.group(3)) -def git_show(commit: str, path: str) -> bytes | None: - """ - Return the bytes of `path` at `commit`, or None if unreadable. - - A blob can be unreadable after a force-push, a history rewrite, or for a - brand-new file. Treat that as "no previous version" rather than failing - every subsequent push. - """ - result = subprocess.run( - ["git", "show", f"{commit}:{path}"], - capture_output=True, - ) - return result.stdout if result.returncode == 0 else None - - def info(message: str) -> None: print(message, file=sys.stderr) -def main(before_commit: str) -> None: +def main() -> None: + changes = json.load(sys.stdin) + bumps: list[dict[str, str]] = [] errors: list[str] = [] - for path in sorted(Path("packages").glob("*/pyproject.toml")): - package = path.parent.name - pyproject = path.as_posix() # git wants forward slashes on every OS - - after = semver(path.read_bytes()) - - before_blob = git_show(before_commit, pyproject) - if before_blob is None: - info(f"No readable {pyproject} at {before_commit}, skipping {package}.") - continue - before = semver(before_blob) + for change in changes: + package = change["package"] + before_raw = change["before"] + after_raw = change["after"] - if after == before: + if before_raw is None or after_raw is None: + info(f"{package}: added or removed, not a release. Skipping.") continue - before_str = ".".join(map(str, before)) - after_str = ".".join(map(str, after)) + before = semver(package, before_raw) + after = semver(package, after_raw) if after < before: - errors.append(f"{package}: {before_str} -> {after_str}") + errors.append(f"{package}: {before_raw} -> {after_raw}") continue - info(f"{package}: {before_str} -> {after_str} (bump)") - bumps.append({"package": package, "version": after_str, "tag": f"{package}-v{after_str}"}) + info(f"{package}: {before_raw} -> {after_raw} (bump)") + bumps.append({"package": package, "version": after_raw, "tag": f"{package}-v{after_raw}"}) if errors: for e in errors: @@ -110,19 +86,6 @@ def main(before_commit: str) -> None: ) sys.exit(1) - # Belt-and-braces re-check of the major-bump cascade (primary enforcement - # is the PR-time version check). Publishing releases with a non-cascaded - # major bump would poison dependents' declared floors. - after_manifests = { - path.parent.name: tomllib.loads(path.read_text(encoding="utf-8")) - for path in sorted(Path("packages").glob("*/pyproject.toml")) - } - violations = check_major_cascade(package_manifests(before_commit), after_manifests) - if violations: - for v in violations: - info(f"::error::{v}") - sys.exit(1) - if not bumps: info("No version bumps detected.") @@ -131,7 +94,4 @@ def main(before_commit: str) -> None: if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} BEFORE_COMMIT", file=sys.stderr) - sys.exit(1) - main(sys.argv[1]) + main() From 7a82ddf5f865b5a0667eb18e2c67c85926742e78 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 14:04:20 -0400 Subject: [PATCH 34/41] [REFACTOR](ci) Split diff and filter into discrete action steps Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/actions/detect-version-bumps/action.yml | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/.github/actions/detect-version-bumps/action.yml b/.github/actions/detect-version-bumps/action.yml index 6e4e65c1f..22cab9ab0 100644 --- a/.github/actions/detect-version-bumps/action.yml +++ b/.github/actions/detect-version-bumps/action.yml @@ -19,21 +19,28 @@ inputs: outputs: count: description: Number of bumped packages. - value: ${{ steps.detect.outputs.count }} + value: ${{ steps.filter.outputs.count }} bumps: description: > JSON array of {"package", "version", "tag"} objects, one per bump. Suitable as a matrix include list. - value: ${{ steps.detect.outputs.bumps }} + value: ${{ steps.filter.outputs.bumps }} runs: using: composite steps: - - name: Detect version bumps - id: detect + - name: Diff package versions + id: diff shell: bash env: BEFORE: ${{ inputs.before }} run: | python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" HEAD \ - | python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" >> "$GITHUB_OUTPUT" + > "${RUNNER_TEMP}/package-version-diff.json" + + - name: Filter to releasable bumps + id: filter + shell: bash + run: | + python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" \ + < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT" From 88e3ac27d1fe7ac11b5512b1cf406fad7ba27053 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 14:07:11 -0400 Subject: [PATCH 35/41] [FEATURE](ci) Enforce floor raises when a workspace dependency majors docs/versioning.md promised that a dependency's major bump requires the dependent to raise its declared floor, but check_major_cascade only checked the dependent's own version. It now also fails when the floor's major stays below the dependency's new major, closing the gap where published metadata would still admit the old breaking-incompatible major. Flagged by @vcschapp in review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/workflows/scripts/package_versions.py | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/package_versions.py b/.github/workflows/scripts/package_versions.py index 108b8247b..c13e7a6f2 100644 --- a/.github/workflows/scripts/package_versions.py +++ b/.github/workflows/scripts/package_versions.py @@ -113,10 +113,11 @@ def check_major_cascade( Workspace dependency floors are declared statically in each package's `project.dependencies`, so a major bump of a dependency is a breaking - change behind every dependent's existing floor. A package whose - direct workspace dependency takes a major bump must therefore take a - major bump in the same change. Checking direct dependencies is enough: - each unbumped link in a longer chain fails its own check. + change behind every dependent's existing floor. A package whose direct + workspace dependency takes a major bump must therefore, in the same + change, take a major bump itself and raise its floor on that dependency + to the new major. Checking direct dependencies is enough: each unbumped + link in a longer chain fails its own check. Returns a list of violation descriptions (empty when compliant). """ @@ -124,6 +125,24 @@ def check_major_cascade( def major(version: str | None) -> int | None: return int(version.split(".")[0]) if version else None + def floor_major(manifests: dict[str, dict], package: str, dep: str) -> int | None: + """Major of `package`'s declared floor on distribution `dep`, if any.""" + manifest = manifests.get(package) + if manifest is None: + return None + for requirement in manifest["project"].get("dependencies", []): + requirement = str(requirement) + match = REQUIREMENT_NAME.match(requirement) + if not match or match.group(1) != dep: + continue + version_match = re.search(r">=\s*(\d+)", requirement) + return int(version_match.group(1)) if version_match else None + return None + + dist_names = { + package: str(m["project"]["name"]) for package, m in after_manifests.items() + } + errors = [] for package, deps in dependency_graph(after_manifests).items(): pkg_before = major(manifest_version(before_manifests, package)) @@ -141,6 +160,13 @@ def major(version: str | None) -> int | None: f"({dep_before}.x -> {dep_after}.x), but {package} does not. " "Major bumps must cascade to dependents." ) + floor = floor_major(after_manifests, package, dist_names[dep]) + if floor is not None and floor < dep_after: + errors.append( + f"{package} declares a floor of {dist_names[dep]}>={floor} " + f"but {dep} is now {dep_after}.x. Raise the floor to the " + "new major so published metadata cannot admit the old one." + ) return errors From b1cac63d482ca73f7bbecb66c8b2f51827e121c8 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 14:18:56 -0400 Subject: [PATCH 36/41] [REFACTOR](ci) Rename workflow to Publish GitHub release Suggested by @vcschapp; the old name described the event, not the action taken. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .github/actions/create-package-release/extract_release_notes.py | 2 +- .github/workflows/release-trigger.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/create-package-release/extract_release_notes.py b/.github/actions/create-package-release/extract_release_notes.py index 90932483b..860214717 100644 --- a/.github/actions/create-package-release/extract_release_notes.py +++ b/.github/actions/create-package-release/extract_release_notes.py @@ -3,7 +3,7 @@ """ Extract one package's release notes from its CHANGELOG.md. -Run by the `Release trigger` workflow. Prints the changelog section for the +Run by the `Publish GitHub release` workflow. Prints the changelog section for the given version (the block from `## []` up to the next `## [` heading, stripped). Prints nothing if the changelog or the section is absent, so the caller can fall back to a default message. diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 0fde5a6b3..bd2519028 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -1,4 +1,4 @@ -name: Release trigger +name: Publish GitHub release # Runs on every push to main that touches any package's pyproject.toml. # For each package whose .. was bumped, cuts a published From db67d7224194c1b7454d2e91135e811ccd77350a Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 14:27:52 -0400 Subject: [PATCH 37/41] [DOCS](versioning) Make the fragment consumption lifecycle explicit towncrier build deletes fragments as it folds them into CHANGELOG.md; the docs never said so, which invited an on-demand-generation reading in review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- docs/versioning.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/versioning.md b/docs/versioning.md index faf56f71d..e288d019f 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -110,6 +110,12 @@ changed package. No tool to install, no config to touch. towncrier only runs at release time, when a maintainer folds the accumulated fragments into `CHANGELOG.md` (see [Cut a release](#cut-a-release)). +> [!NOTE] +> `towncrier build` consumes fragments: it deletes them from `changelog.d/` +> as it folds them into `CHANGELOG.md`. The fragment directory only ever +> holds unreleased changes, and the committed `CHANGELOG.md` is the sole +> durable record of past releases. + ### Add a changelog fragment Release notes are assembled from From 7a8e9592ad2ff58c7d4fcac9f91e2890abfe03da Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 14:28:31 -0400 Subject: [PATCH 38/41] [DOCS](versioning) State the CI-never-writes-to-main principle Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- docs/versioning.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/versioning.md b/docs/versioning.md index e288d019f..c0ee1058a 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -187,3 +187,12 @@ flowchart LR - **towncrier fragments.** Notes are written in context per PR and assembled automatically, with no merge conflicts on a shared changelog and no hand-written notes at release time. +- **CI never writes to `main`.** Every commit on `main` arrives through a + reviewed PR; CI is a pure reader. Automation that commits back to `main` + needs a bot identity with branch-protection bypass (a compromised workflow + can then push arbitrary code), skip-guards against re-triggering + push-driven workflows on its own commits, and retry logic for races with + human merges, and every synthetic commit is an unreviewed change on the + protected branch. Designs that require a write-back (e.g. on-demand + changelog generation with post-release fragment cleanup) are rejected on + this principle. From 11a111054320818f1aceeff3d3f46258420c9fc9 Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 15:26:15 -0400 Subject: [PATCH 39/41] [FEATURE](deps) Take towncrier as a dev dependency Pins the changelog tool the release process depends on instead of pulling latest via uvx at each invocation; all documented invocations become uv run towncrier. Suggested by @sethfitz in review. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../actions/create-package-release/action.yml | 2 +- .github/workflows/release-trigger.yaml | 2 +- .../workflows/require-changelog-fragment.yaml | 2 +- CONTRIBUTING.md | 2 +- docs/versioning.md | 4 ++-- pyproject.toml | 5 +++-- uv.lock | 18 +++++++++++++++++- 7 files changed, 26 insertions(+), 9 deletions(-) diff --git a/.github/actions/create-package-release/action.yml b/.github/actions/create-package-release/action.yml index 0f92d1b3c..57c6ad38e 100644 --- a/.github/actions/create-package-release/action.yml +++ b/.github/actions/create-package-release/action.yml @@ -74,7 +74,7 @@ runs: changelog="packages/${PACKAGE}/CHANGELOG.md" notes=$(python3 "${GITHUB_ACTION_PATH}/extract_release_notes.py" "$VERSION" "$changelog") if [ -z "$notes" ]; then - notes="Release ${VERSION} of \`${PACKAGE}\`. No changelog section was found; add towncrier fragments under \`packages/${PACKAGE}/changelog.d\` and run \`uvx towncrier build --config pyproject.toml --dir packages/${PACKAGE}\`." + notes="Release ${VERSION} of \`${PACKAGE}\`. No changelog section was found; add towncrier fragments under \`packages/${PACKAGE}/changelog.d\` and run \`uv run towncrier build --config pyproject.toml --dir packages/${PACKAGE}\`." fi printf '%s\n' "$notes" > "${RUNNER_TEMP}/notes.md" diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index bd2519028..068681e20 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -9,7 +9,7 @@ name: Publish GitHub release # `overture-schema` release is marked "Latest"; all others are not. # # Release notes come from towncrier: a release PR bumps the version and runs -# `uvx towncrier build`, folding that package's changelog.d/ fragments into its +# `uv run towncrier build`, folding that package's changelog.d/ fragments into its # CHANGELOG.md (reviewed in the PR). This workflow reads back the section. # # All three version components are human-owned; any increase (patch included) diff --git a/.github/workflows/require-changelog-fragment.yaml b/.github/workflows/require-changelog-fragment.yaml index 861d6e021..2a7c249ba 100644 --- a/.github/workflows/require-changelog-fragment.yaml +++ b/.github/workflows/require-changelog-fragment.yaml @@ -3,7 +3,7 @@ name: Changelog fragment verification # Enforces the changelog policy: any PR that changes a package must also carry # that package's changelog update, either a new towncrier fragment under # changelog.d/, or a built CHANGELOG.md (the result of running -# `uvx towncrier build`). +# `uv run towncrier build`). # # "Changes a package" means any file under packages// other than that # package's own changelog artifacts (changelog.d/ and CHANGELOG.md). Every diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b8fd5805c..6270406a7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -134,7 +134,7 @@ merge to `main`; run `git pull --rebase` before pushing again. [changelog quick start](docs/versioning.md#changelog-quick-start). Add one under `packages//changelog.d/` and run - `uvx towncrier build --config pyproject.toml --dir packages/`. CI + `uv run towncrier build --config pyproject.toml --dir packages/`. CI enforces it. Full version scheme, tag scheme, and release flow: [docs/versioning.md](docs/versioning.md). diff --git a/docs/versioning.md b/docs/versioning.md index c0ee1058a..0f6e3d134 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -140,7 +140,7 @@ The file body is the note itself, written in past tense ```bash # from the repo root -uvx towncrier build --config pyproject.toml --dir packages/ --draft --version +uv run towncrier build --config pyproject.toml --dir packages/ --draft --version ``` A fragment (or an already-built `CHANGELOG.md` entry) is required on any PR that @@ -154,7 +154,7 @@ changes that package, whether or not it bumps the version. ### Cut a release 1. Bump the version in the package's `pyproject.toml`, - then run `uvx towncrier build --config pyproject.toml --dir packages/` + then run `uv run towncrier build --config pyproject.toml --dir packages/` from the repo root to fold its fragments into `CHANGELOG.md`. Patch and minor bumps target `main`; major bumps go via `vnext` and reach `main` through a release merge. diff --git a/pyproject.toml b/pyproject.toml index 7d8488534..8f0243aa2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dev = [ "pytest-cov>=7.0.0", "pytest-testmon>=2.2.0", "ruff>=0.13.0", + "towncrier>=25.8.0", ] [tool.pytest.ini_options] @@ -81,7 +82,7 @@ verbosity_subtests = 0 # Shared changelog (towncrier) config for every package under packages/*. # Build a package's notes from the repo root: -# uvx towncrier build --config pyproject.toml --dir packages/ --version ..0 +# uv run towncrier build --config pyproject.toml --dir packages/ --version ..0 # A package may override these categories by adding its own [tool.towncrier] # block and building from that package directory (towncrier replaces, not merges). [tool.towncrier] @@ -114,4 +115,4 @@ showcontent = true [[tool.towncrier.type]] directory = "misc" name = "Miscellaneous" -showcontent = true \ No newline at end of file +showcontent = true diff --git a/uv.lock b/uv.lock index f1eb70fa3..d5ae8be1b 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. +exclude-newer = "2026-07-28T19:24:56.7525677Z" exclude-newer-span = "P1W" [manifest] @@ -1214,6 +1214,7 @@ dev = [ { name = "pytest-cov" }, { name = "pytest-testmon" }, { name = "ruff" }, + { name = "towncrier" }, ] [package.metadata] @@ -1227,6 +1228,7 @@ dev = [ { name = "pytest-cov", specifier = ">=7.0.0" }, { name = "pytest-testmon", specifier = ">=2.2.0" }, { name = "ruff", specifier = ">=0.13.0" }, + { name = "towncrier", specifier = ">=25.8.0" }, ] [[package]] @@ -1725,6 +1727,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "towncrier" +version = "25.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "jinja2" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/eb/5bf25a34123698d3bbab39c5bc5375f8f8bcbcc5a136964ade66935b8b9d/towncrier-25.8.0.tar.gz", hash = "sha256:eef16d29f831ad57abb3ae32a0565739866219f1ebfbdd297d32894eb9940eb1", size = 76322, upload-time = "2025-08-30T11:41:55.393Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/06/8ba22ec32c74ac1be3baa26116e3c28bc0e76a5387476921d20b6fdade11/towncrier-25.8.0-py3-none-any.whl", hash = "sha256:b953d133d98f9aeae9084b56a3563fd2519dfc6ec33f61c9cd2c61ff243fb513", size = 65101, upload-time = "2025-08-30T11:41:53.644Z" }, +] + [[package]] name = "types-pyyaml" version = "6.0.12.20260518" From 5d5895cdb3beeec260924e9e09a1e1664b399bcc Mon Sep 17 00:00:00 2001 From: John McCall Date: Tue, 4 Aug 2026 16:21:57 -0400 Subject: [PATCH 40/41] [FEATURE](ci) Continue the legacy bare tag series for umbrella releases Umbrella overture-schema releases now also create a bare v vanity git tag at the release commit, with no second GitHub Release attached (the secondary release provides nothing of value). The bare series is the convention consumers of the primary entrypoint already know. Collisions on the vanity tag warn rather than failing the already-published release. Consensus from review with @sethfitz and @vcschapp. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../actions/create-package-release/action.yml | 33 ++++++++++++++++++- .github/workflows/release-trigger.yaml | 3 ++ docs/versioning.md | 11 ++++--- 3 files changed, 41 insertions(+), 6 deletions(-) diff --git a/.github/actions/create-package-release/action.yml b/.github/actions/create-package-release/action.yml index 57c6ad38e..624be3bb1 100644 --- a/.github/actions/create-package-release/action.yml +++ b/.github/actions/create-package-release/action.yml @@ -6,7 +6,8 @@ description: > Reads the `## []` section of `packages//CHANGELOG.md` for the release notes (falling back to a pointer message if absent), fails if the target tag already exists, and creates the release tagged - `-v` at `target`. + `-v` at `target`. Optionally also creates a bare vanity + tag (no second GitHub Release) to continue a legacy tag series. Prerequisites: repo must be checked out and the GitHub CLI (`gh`) available (both true on GitHub-hosted runners). @@ -28,6 +29,14 @@ inputs: description: Whether to mark this release as "Latest" (true/false). required: false default: "false" + vanity-tag: + description: > + Optional additional bare git tag to create at `target` (e.g. v1.18.0), + with no GitHub Release attached. A pre-existing tag warns instead of + failing; the vanity tag is cosmetic and must not fail a completed + release. + required: false + default: "" dry-run: description: > When "true", resolve and print what would be released without creating @@ -57,6 +66,7 @@ runs: TAG: ${{ inputs.tag }} TARGET: ${{ inputs.target }} LATEST: ${{ inputs.latest }} + VANITY_TAG: ${{ inputs.vanity-tag }} DRY_RUN: ${{ inputs.dry-run }} run: | set -euo pipefail @@ -87,6 +97,9 @@ runs: echo "## 🔍 Dry run: \`${PACKAGE}\` ${VERSION}" echo "" echo "Would create tag \`${TAG}\` at \`${TARGET}\` (${latest_flag})." + if [ -n "$VANITY_TAG" ]; then + echo "Would also create vanity tag \`${VANITY_TAG}\` (no release)." + fi echo "" echo "
Notes" echo "" @@ -108,8 +121,26 @@ runs: url=$(gh release view "$TAG" --repo "$GITHUB_REPOSITORY" --json url --jq .url) echo "release-url=${url}" >> "$GITHUB_OUTPUT" + # Vanity tag: a bare ref continuing the legacy series, no second + # release. Cosmetic, so a collision warns rather than failing the + # already-published release. + if [ -n "$VANITY_TAG" ]; then + if gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${VANITY_TAG}" >/dev/null 2>&1; then + echo "::warning::Vanity tag ${VANITY_TAG} already exists; leaving it untouched." + else + gh api "repos/${GITHUB_REPOSITORY}/git/refs" \ + -f ref="refs/tags/${VANITY_TAG}" \ + -f sha="$TARGET" >/dev/null + echo "Created vanity tag ${VANITY_TAG} at ${TARGET}." + fi + fi + { echo "## 📦 Released \`${PACKAGE}\` ${VERSION}" echo "" echo "Tag \`${TAG}\`: [published GitHub Release](${url})." + if [ -n "$VANITY_TAG" ]; then + echo "" + echo "Vanity tag \`${VANITY_TAG}\` continues the legacy series (no separate release)." + fi } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/release-trigger.yaml b/.github/workflows/release-trigger.yaml index 068681e20..b54b7b75c 100644 --- a/.github/workflows/release-trigger.yaml +++ b/.github/workflows/release-trigger.yaml @@ -80,4 +80,7 @@ jobs: tag: ${{ matrix.tag }} target: ${{ github.event.after }} latest: ${{ matrix.package == 'overture-schema' }} + # The umbrella package continues the legacy bare tag series as a + # vanity tag (no second release); see docs/versioning.md. + vanity-tag: ${{ matrix.package == 'overture-schema' && format('v{0}', matrix.version) || '' }} token: ${{ github.token }} diff --git a/docs/versioning.md b/docs/versioning.md index 0f6e3d134..f1b6537fc 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -70,11 +70,12 @@ cascade (see [Guardrails](#guardrails)). Each package has its own release series: tag `-v..`, title `` `` ``. The umbrella `overture-schema` release is -flagged **Latest**. - -Historical single-series tags (`v0.4.0` … `v1.17.0`) remain valid. The -package-prefixed scheme is new so packages can version independently. This is a -deliberate, one-time discontinuity. +flagged **Latest**, and additionally continues the historical bare series +(`v0.4.0` … `v1.17.0`) as a vanity tag: each umbrella release also creates a +bare `v` git tag at the same commit, with no second GitHub Release +attached. The umbrella package is the primary entrypoint for most consumers, +so its bare tags keep the long-standing convention alive; all other packages +use only the package-prefixed scheme. ### Guardrails From 6836a3bb1861561af4a96ad0b5560fe0d8e09574 Mon Sep 17 00:00:00 2001 From: John McCall Date: Wed, 5 Aug 2026 11:08:28 -0400 Subject: [PATCH 41/41] [DOCS](changelog) Require --version in every towncrier invocation towncrier only discovers a version from its config file (version or package key), and the shared root config deliberately has neither, so --version is mandatory in our setup (verified: omitting it errors). Also drops the towncrier command from the contributor-facing CONTRIBUTING bullet, folding fragments is a release-time step, not part of a normal PR. Plus the .0 -> . nit in the root config comment. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Signed-off-by: John McCall --- .../actions/create-package-release/action.yml | 2 +- CONTRIBUTING.md | 10 ++++------ docs/versioning.md | 20 ++++++++++++++----- pyproject.toml | 2 +- 4 files changed, 21 insertions(+), 13 deletions(-) diff --git a/.github/actions/create-package-release/action.yml b/.github/actions/create-package-release/action.yml index 624be3bb1..d18fe1322 100644 --- a/.github/actions/create-package-release/action.yml +++ b/.github/actions/create-package-release/action.yml @@ -84,7 +84,7 @@ runs: changelog="packages/${PACKAGE}/CHANGELOG.md" notes=$(python3 "${GITHUB_ACTION_PATH}/extract_release_notes.py" "$VERSION" "$changelog") if [ -z "$notes" ]; then - notes="Release ${VERSION} of \`${PACKAGE}\`. No changelog section was found; add towncrier fragments under \`packages/${PACKAGE}/changelog.d\` and run \`uv run towncrier build --config pyproject.toml --dir packages/${PACKAGE}\`." + notes="Release ${VERSION} of \`${PACKAGE}\`. No changelog section was found; add towncrier fragments under \`packages/${PACKAGE}/changelog.d\` and run \`uv run towncrier build --config pyproject.toml --dir packages/${PACKAGE} --version ${VERSION}\`." fi printf '%s\n' "$notes" > "${RUNNER_TEMP}/notes.md" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6270406a7..cf31d7a4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -130,11 +130,9 @@ merge to `main`; run `git pull --rebase` before pushing again. `overture-schema`, which pulls in the theme and support packages for a coherent set. - Any change to a package **requires a changelog fragment**: one sentence in - one file, see the - [changelog quick start](docs/versioning.md#changelog-quick-start). Add one - under - `packages//changelog.d/` and run - `uv run towncrier build --config pyproject.toml --dir packages/`. CI - enforces it. + one file under `packages//changelog.d/`, see the + [changelog quick start](docs/versioning.md#changelog-quick-start). CI + enforces it. Fragments are folded into `CHANGELOG.md` at release time, not + in your PR. Full version scheme, tag scheme, and release flow: [docs/versioning.md](docs/versioning.md). diff --git a/docs/versioning.md b/docs/versioning.md index f1b6537fc..88a31f995 100644 --- a/docs/versioning.md +++ b/docs/versioning.md @@ -154,11 +154,21 @@ changes that package, whether or not it bumps the version. ### Cut a release -1. Bump the version in the package's `pyproject.toml`, - then run `uv run towncrier build --config pyproject.toml --dir packages/` - from the repo root to fold its fragments into `CHANGELOG.md`. Patch and minor - bumps target `main`; major bumps go via `vnext` and reach `main` through a - release merge. +1. Bump the version in the package's `pyproject.toml`, then fold its fragments + into `CHANGELOG.md` from the repo root: + + ```bash + uv run towncrier build --config pyproject.toml --dir packages/ --version + ``` + + `--version` is required: towncrier only discovers a version from its + [config file](https://towncrier.readthedocs.io/en/stable/configuration.html) + (`version` or `package` key), and our config is the shared root + `pyproject.toml`, which deliberately has neither so one number can't stamp + every package. Pass the version you just bumped to. + + Patch and minor bumps target `main`; major bumps go via `vnext` and reach + `main` through a release merge. 2. On merge to `main`, `release-trigger` publishes one GitHub Release per bumped package: tag `-v`, notes from that package's `CHANGELOG.md`. diff --git a/pyproject.toml b/pyproject.toml index 8f0243aa2..ae02f5090 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ verbosity_subtests = 0 # Shared changelog (towncrier) config for every package under packages/*. # Build a package's notes from the repo root: -# uv run towncrier build --config pyproject.toml --dir packages/ --version ..0 +# uv run towncrier build --config pyproject.toml --dir packages/ --version .. # A package may override these categories by adding its own [tool.towncrier] # block and building from that package directory (towncrier replaces, not merges). [tool.towncrier]