[FEATURE] Branching strategy Phase 3 - PyPI and CodeArtifact publish workflows

.github/actions/detect-affected-packages/action.yml

@@ -0,0 +1,67 @@
+name: Detect affected packages
+description: >
+  Detects packages affected by a no-bump push to main, i.e. any file changed
+  under packages/<pkg>/ between two commits, excluding packages whose
+  pyproject.toml version was also bumped in the same range (those release via
+  release-trigger instead; see docs/versioning.md) and packages removed in the
+  range (nothing left to build).
+
+  Prerequisites: repo must be checked out with `fetch-depth: 0` so `before` is
+  reachable.
+
+inputs:
+  before:
+    description: The base commit SHA to compare against (e.g. github.event.before).
+    required: true
+  after:
+    description: The head commit SHA to compare to. Defaults to the checked-out HEAD.
+    required: false
+    default: HEAD
+
+outputs:
+  count:
+    description: Number of affected packages.
+    value: ${{ steps.filter.outputs.count }}
+  packages:
+    description: >
+      JSON array of affected package directory names, e.g.
+      ["overture-schema-common"]. Suitable as a matrix input.
+    value: ${{ steps.filter.outputs.packages }}
+
+runs:
+  using: composite
+  steps:
+    - name: List changed package directories
+      id: changed-dirs
+      uses: tj-actions/changed-files@9426d40962ed5378910ee2e21d5f8c6fcbf2dd96 # v47.0.6
+      with:
+        base_sha: ${{ inputs.before }}
+        sha: ${{ inputs.after }}
+        files: packages/**
+        dir_names: true
+        dir_names_max_depth: 2
+        matrix: true
+
+    # Independent of the step above (neither reads the other's output); runs
+    # after it only so both feed the final filter step back to back.
+    - name: Diff package versions
+      id: diff
+      uses: ./.github/actions/diff-package-versions
+      with:
+        before: ${{ inputs.before }}
+        after: ${{ inputs.after }}
+
+    - name: Detect affected packages
+      id: filter
+      shell: bash
+      env:
+        # all_modified_files (not all_changed_files) so deletions count too: a
+        # file removed from a package without a version bump still changes
+        # the built wheel.
+        CHANGED_DIRS: ${{ steps.changed-dirs.outputs.all_modified_files }}
+        DIFF: ${{ steps.diff.outputs.diff }}
+      run: |
+        set -euo pipefail
+        jq -n --argjson version_diff "$DIFF" --argjson changed_dirs "$CHANGED_DIRS" \
+          '{version_diff: $version_diff, changed_dirs: $changed_dirs}' \
+          | python3 "${GITHUB_ACTION_PATH}/detect_affected_packages.py" >> "$GITHUB_OUTPUT"

.github/actions/detect-affected-packages/detect_affected_packages.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+
+"""
+Detect packages affected by a no-bump push to main.
+
+Composes `package_versions.py diff`'s version-diff JSON with a JSON array of
+changed package directories (tj-actions/changed-files' `dir_names` output) to
+find packages whose directory changed without also changing their
+pyproject.toml version. Those need an internal `.postN` build (see
+docs/versioning.md). Packages with a version bump in the same range are
+excluded because they release via `release-trigger` instead, and removed
+packages are excluded because there is nothing left to build.
+
+Reads a single JSON object from stdin: {"version_diff": [...], "changed_dirs":
+[...]}. Run from the repository root:
+
+    jq -n \\
+        --argjson version_diff "$(python3 package_versions.py diff BEFORE AFTER)" \\
+        --argjson changed_dirs '["packages/overture-schema-common"]' \\
+        '{version_diff: $version_diff, changed_dirs: $changed_dirs}' \\
+      | python3 detect_affected_packages.py
+
+Prints `$GITHUB_OUTPUT` lines on stdout (progress goes to stderr):
+
+    count=<n>        Number of affected packages.
+    packages=<json>  JSON array of affected package directory names, e.g.
+                     ["overture-schema-common"]. Suitable as a matrix input.
+
+Exit status:
+    0  Success (including the no-affected case).
+    1  Malformed input: a changed_dirs entry isn't packages/<pkg>.
+"""
+
+import json
+import sys
+
+
+def _package_name(path: str) -> str:
+    """Extract <pkg> from a packages/<pkg> changed-directory entry."""
+    parts = path.split("/", 1)
+    if len(parts) < 2:
+        raise ValueError(
+            f"Expected a 'packages/<pkg>' changed-directory entry, got {path!r}. "
+            "Check dir_names_max_depth on the changed-files step."
+        )
+    return parts[1]
+
+
+def main() -> None:
+    payload = json.load(sys.stdin)
+    version_diff = payload["version_diff"]
+    changed_paths = payload["changed_dirs"]
+
+    bumped = {change["package"] for change in version_diff if change["after"] is not None}
+    removed = {change["package"] for change in version_diff if change["after"] is None}
+
+    changed = {_package_name(path) for path in changed_paths}
+    affected = sorted(changed - bumped - removed)
+
+    for package in sorted(changed):
+        if package in removed:
+            print(f"{package}: removed. Skipping.", file=sys.stderr)
+        elif package in bumped:
+            print(
+                f"{package}: version bumped, handled by release-trigger. "
+                "Skipping internal build.",
+                file=sys.stderr,
+            )
+        else:
+            print(f"{package}: changed, no bump -> internal build.", file=sys.stderr)
+
+    if not affected:
+        print("No affected packages (no unreleased changes to publish).", file=sys.stderr)
+
+    print(f"count={len(affected)}")
+    print(f"packages={json.dumps(affected)}")
+
+
+if __name__ == "__main__":
+    main()

.github/actions/detect-version-bumps/action.yml

@@ -31,16 +31,14 @@ runs:
   steps:
     - name: Diff package versions
       id: diff
-      shell: bash
-      env:
-        BEFORE: ${{ inputs.before }}
-      run: |
-        python3 ./.github/workflows/scripts/package_versions.py diff "$BEFORE" HEAD \
-          > "${RUNNER_TEMP}/package-version-diff.json"
+      uses: ./.github/actions/diff-package-versions
+      with:
+        before: ${{ inputs.before }}
 
     - name: Filter to releasable bumps
       id: filter
       shell: bash
+      env:
+        DIFF: ${{ steps.diff.outputs.diff }}
       run: |
-        python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" \
-          < "${RUNNER_TEMP}/package-version-diff.json" >> "$GITHUB_OUTPUT"
+        echo "$DIFF" | python3 "${GITHUB_ACTION_PATH}/detect_version_bumps.py" >> "$GITHUB_OUTPUT"

.github/actions/detect-version-bumps/detect_version_bumps.py

@@ -17,8 +17,9 @@
   - Released versions must be plain `<major>.<minor>.<patch>`; PEP 440
     variants like `1.2.3rc4` fail loudly.
   - A version decrease fails: it must never land on main.
-  - Added packages (`before` null) and removed packages (`after` null) are
-    not releases; they are skipped.
+  - Added packages (`before` null) count as a bump: a package's first
+    version releases to PyPI like any other. Removed packages (`after`
+    null) are not releases; they are skipped.
 
 The `detect-version-bumps` action composes this with `package_versions.py`,
 which owns reading versions from git (and enforces the major-bump cascade
@@ -64,13 +65,19 @@ def main() -> None:
         before_raw = change["before"]
         after_raw = change["after"]
 
-        if before_raw is None or after_raw is None:
-            info(f"{package}: added or removed, not a release. Skipping.")
+        if after_raw is None:
+            info(f"{package}: removed, not a release. Skipping.")
             continue
 
-        before = semver(package, before_raw)
         after = semver(package, after_raw)
 
+        if before_raw is None:
+            info(f"{package}: new package at {after_raw} (bump)")
+            bumps.append({"package": package, "version": after_raw, "tag": f"{package}-v{after_raw}"})
+            continue
+
+        before = semver(package, before_raw)
+
         if after < before:
             errors.append(f"{package}: {before_raw} -> {after_raw}")
             continue

.github/actions/diff-package-versions/action.yml

@@ -0,0 +1,46 @@
+name: Diff package versions
+description: >
+  Diffs packages/*/pyproject.toml versions between two commits (see
+  package_versions.py), reading blobs directly from git. Enforces the
+  major-bump cascade: a package whose direct workspace dependency takes a
+  major bump must take one itself; the action fails otherwise.
+
+  Prerequisites: repo must be checked out with `fetch-depth: 0` so `before`
+  is reachable.
+
+inputs:
+  before:
+    description: The base commit SHA to compare against (e.g. github.event.before).
+    required: true
+  after:
+    description: The head commit SHA to compare to. Defaults to the checked-out HEAD.
+    required: false
+    default: HEAD
+
+outputs:
+  count:
+    description: Number of packages with a version change (added, removed, or bumped).
+    value: ${{ steps.diff.outputs.count }}
+  diff:
+    description: >
+      JSON array of {"package", "before", "after"} objects, topologically
+      sorted (dependencies before dependents). `before`/`after` are null when
+      the package didn't exist at that commit.
+    value: ${{ steps.diff.outputs.diff }}
+
+runs:
+  using: composite
+  steps:
+    - name: Diff package versions
+      id: diff
+      shell: bash
+      env:
+        BEFORE: ${{ inputs.before }}
+        AFTER: ${{ inputs.after }}
+      run: |
+        python3 "${GITHUB_ACTION_PATH}/package_versions.py" diff "$BEFORE" "$AFTER" \
+          > "${RUNNER_TEMP}/package-version-diff.json"
+        {
+          echo "count=$(jq 'length' "${RUNNER_TEMP}/package-version-diff.json")"
+          echo "diff=$(jq -c . "${RUNNER_TEMP}/package-version-diff.json")"
+        } >> "$GITHUB_OUTPUT"

.github/workflows/scripts/package_versions.py → .github/actions/diff-package-versions/package_versions.py

@@ -3,9 +3,11 @@
 """
 Diff per-package versions between two git commits.
 
-Run from the repository root:
+Run with the repository root as the working directory (git commands need it
+as CWD); the script itself can live anywhere, e.g. the `diff-package-versions`
+action invokes it via `$GITHUB_ACTION_PATH`:
 
-    python3 package_versions.py diff <before-commit> <after-commit>
+    python3 path/to/package_versions.py diff <before-commit> <after-commit>
 
 Reads each `packages/*/pyproject.toml` blob directly from git at both commits
 (no checkout switching, no environment sync) and prints the packages whose

.github/workflows/check-python-code.yaml

@@ -29,13 +29,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Default resolution exercises the committed lock against every
-        # supported Python minor version. The lowest-direct cell pins each
-        # direct dependency to its declared floor (see UV_RESOLUTION below)
-        # and runs only on the Python floor, since the resolved-low pyspark
-        # 3.4 wheels exist for 3.10/3.11 only.
+        # locked exercises the committed lock against every supported Python
+        # minor version. The lowest-direct cell pins each direct dependency
+        # to its declared floor (see UV_RESOLUTION below) and runs only on
+        # the Python floor, since the resolved-low pyspark 3.4 wheels exist
+        # for 3.10/3.11 only.
         python: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-        resolution: [default]
+        resolution: [locked]
         include:
           - python: "3.10"
             resolution: lowest-direct
@@ -80,11 +80,11 @@ jobs:
         run: echo "UV_RESOLUTION=lowest-direct" >> "$GITHUB_ENV"
 
       # Fail fast if uv.lock is stale (e.g. a pyproject.toml version bump
-      # landed without a matching `uv lock` run). Only for the default
+      # landed without a matching `uv lock` run). Only for the locked
       # resolution cells -- lowest-direct intentionally re-resolves away
       # from the committed lock, so `--locked` would always fail there.
       - name: Configure lock check
-        if: matrix.resolution == 'default'
+        if: matrix.resolution == 'locked'
         run: echo "UV_LOCKED=1" >> "$GITHUB_ENV"
 
       - name: Run make check