From 1e73b44c424e12a3e034a461a7ba498cb5a2ad98 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Mon, 28 Sep 2026 08:41:07 -0700 Subject: [PATCH 1/3] fix(ci): validate release source before building distributions --- .github/workflows/publish.yml | 10 ++- tests/test_repository_workflow_interfaces.py | 67 +++++++++++++++++++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 4eda6f203c..7e9818727c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -52,11 +52,17 @@ jobs: version: "0.11.14" enable-cache: false python-version: "3.14" - - name: Install dependencies + - name: Check release source working-directory: release-source + shell: bash env: OPENAI_API_KEY: fake-for-tests - run: make sync + UV_FROZEN: "1" + run: | + for python_version in 3.10 3.11 3.12 3.13 3.14; do + UV_PYTHON="$python_version" make sync tests + done + make typecheck - name: Build package working-directory: release-source run: uv build diff --git a/tests/test_repository_workflow_interfaces.py b/tests/test_repository_workflow_interfaces.py index 623a7f8517..4b78ccff98 100644 --- a/tests/test_repository_workflow_interfaces.py +++ b/tests/test_repository_workflow_interfaces.py @@ -1,9 +1,15 @@ from __future__ import annotations +import os import re import runpy +import shutil +import subprocess from pathlib import Path +import pytest +import yaml + ROOT = Path(__file__).resolve().parents[1] MAKEFILE = ROOT / "Makefile" TESTS_WORKFLOW = ROOT / ".github" / "workflows" / "tests.yml" @@ -180,11 +186,70 @@ def test_release_build_validates_before_executing_candidate_code() -> None: assert build.count("persist-credentials: false") == 2 assert "fetch-depth: 0" in build validation = build.index("python -I control/.github/scripts/verify_release.py") - assert validation < build.index("run: make sync") < build.index("run: uv build") + assert validation < build.index('UV_PYTHON="$python_version" make sync tests') + assert build.index("make typecheck") < build.index("run: uv build") assert ' --tag "$RELEASE_TAG" --expected-sha "$RELEASE_SHA"' in build assert "enable-cache: false" in build +@pytest.mark.parametrize("failed_check", [None, "3.12:sync tests", ":typecheck"]) +def test_release_checks_fail_closed(tmp_path: Path, failed_check: str | None) -> None: + bash = shutil.which("bash") + if bash is None: + pytest.skip("The publish workflow requires Bash.") + workflow = yaml.safe_load(PUBLISH_WORKFLOW.read_text(encoding="utf-8")) + build = workflow["jobs"]["build"] + steps = build["steps"] + check_index = next(i for i, step in enumerate(steps) if step["name"] == "Check release source") + check = steps[check_index] + package_index = next(i for i, step in enumerate(steps) if step.get("run") == "uv build") + assert check_index < package_index + assert ( + check["working-directory"] == steps[package_index]["working-directory"] == "release-source" + ) + assert check["shell"] == "bash" + assert check["env"] == {"OPENAI_API_KEY": "fake-for-tests", "UV_FROZEN": "1"} + assert "if" not in build and "continue-on-error" not in build + for step in steps[check_index:]: + assert "if" not in step and "continue-on-error" not in step + + bin_dir = tmp_path / "bin" + bin_dir.mkdir() + make = bin_dir / "make" + make.write_text( + "#!/bin/sh\n" + 'check="${UV_PYTHON:-}:$*"\n' + 'printf "%s\\n" "$check" >> "$CHECK_LOG"\n' + '[ "$check" != "$FAILED_CHECK" ]\n', + encoding="utf-8", + ) + make.chmod(0o755) + log = tmp_path / "checks.log" + result = subprocess.run( + [bash, "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", check["run"]], + cwd=tmp_path, + env={ + "PATH": f"{bin_dir}{os.pathsep}{os.defpath}", + "CHECK_LOG": str(log), + "FAILED_CHECK": failed_check or "", + **check["env"], + }, + capture_output=True, + text=True, + timeout=5, + ) + expected = [f"3.{minor}:sync tests" for minor in range(10, 15)] + [":typecheck"] + if failed_check is None: + assert result.returncode == 0, result.stderr + assert log.read_text(encoding="utf-8").splitlines() == expected + else: + assert result.returncode != 0 + assert ( + log.read_text(encoding="utf-8").splitlines() + == expected[: expected.index(failed_check) + 1] + ) + + def test_pypi_job_only_publishes_the_build_artifact() -> None: workflow = PUBLISH_WORKFLOW.read_text(encoding="utf-8") publish = _workflow_job("publish", PUBLISH_WORKFLOW) From 20271c1b895c5901417b00737668cc2a28fccad5 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Mon, 28 Sep 2026 09:15:22 -0700 Subject: [PATCH 2/3] fix(ci): reject stale dependency locks during release validation --- .github/workflows/publish.yml | 2 +- tests/test_repository_workflow_interfaces.py | 50 +++++++++++++++++++- 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 7e9818727c..366d28ca2d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -57,7 +57,7 @@ jobs: shell: bash env: OPENAI_API_KEY: fake-for-tests - UV_FROZEN: "1" + UV_LOCKED: "1" run: | for python_version in 3.10 3.11 3.12 3.13 3.14; do UV_PYTHON="$python_version" make sync tests diff --git a/tests/test_repository_workflow_interfaces.py b/tests/test_repository_workflow_interfaces.py index 4b78ccff98..4032aef3ff 100644 --- a/tests/test_repository_workflow_interfaces.py +++ b/tests/test_repository_workflow_interfaces.py @@ -5,6 +5,7 @@ import runpy import shutil import subprocess +import sys from pathlib import Path import pytest @@ -208,7 +209,7 @@ def test_release_checks_fail_closed(tmp_path: Path, failed_check: str | None) -> check["working-directory"] == steps[package_index]["working-directory"] == "release-source" ) assert check["shell"] == "bash" - assert check["env"] == {"OPENAI_API_KEY": "fake-for-tests", "UV_FROZEN": "1"} + assert check["env"] == {"OPENAI_API_KEY": "fake-for-tests", "UV_LOCKED": "1"} assert "if" not in build and "continue-on-error" not in build for step in steps[check_index:]: assert "if" not in step and "continue-on-error" not in step @@ -250,6 +251,53 @@ def test_release_checks_fail_closed(tmp_path: Path, failed_check: str | None) -> ) +def test_release_checks_reject_stale_lockfile(tmp_path: Path) -> None: + uv = shutil.which("uv") + if uv is None: + pytest.skip("The publish workflow requires uv.") + workflow = yaml.safe_load(PUBLISH_WORKFLOW.read_text(encoding="utf-8")) + check = next( + step + for step in workflow["jobs"]["build"]["steps"] + if step["name"] == "Check release source" + ) + dependency = tmp_path / "dependency" + dependency.mkdir() + (dependency / "pyproject.toml").write_text( + '[project]\nname = "fixture-dependency"\nversion = "0.1.0"\n', encoding="utf-8" + ) + project = ( + '[project]\nname = "release-fixture"\nversion = "0.1.0"\n' + 'requires-python = ">=3.10"\ndependencies = []\n' + '[tool.uv.sources]\nfixture-dependency = { path = "dependency" }\n' + ) + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text(project, encoding="utf-8") + env = { + "PATH": os.defpath, + "UV_PYTHON": sys.executable, + "UV_CACHE_DIR": str(tmp_path / "cache"), + } + if "SYSTEMROOT" in os.environ: + env["SYSTEMROOT"] = os.environ["SYSTEMROOT"] + command = [uv, "--offline", "--no-config"] + subprocess.run(command + ["lock"], cwd=tmp_path, env=env, check=True, timeout=15) + lockfile = tmp_path / "uv.lock" + original_lock = lockfile.read_bytes() + # Exercise synchronization without building or installing either fixture package. + sync = command + ["sync", "--no-install-project", "--no-install-package", "fixture-dependency"] + env.update(check["env"]) + subprocess.run(sync, cwd=tmp_path, env=env, check=True, timeout=15) + pyproject.write_text( + project.replace("dependencies = []", 'dependencies = ["fixture-dependency"]'), + encoding="utf-8", + ) + result = subprocess.run(sync, cwd=tmp_path, env=env, capture_output=True, text=True, timeout=15) + assert result.returncode != 0 + assert "lockfile" in result.stderr and "needs to be updated" in result.stderr + assert lockfile.read_bytes() == original_lock + + def test_pypi_job_only_publishes_the_build_artifact() -> None: workflow = PUBLISH_WORKFLOW.read_text(encoding="utf-8") publish = _workflow_job("publish", PUBLISH_WORKFLOW) From 550787c5116e411820a5a40b381f86419bf423f5 Mon Sep 17 00:00:00 2001 From: Justin Beckwith Date: Mon, 28 Sep 2026 09:30:24 -0700 Subject: [PATCH 3/3] fix(ci): isolate release checks from distribution builds --- .github/workflows/publish.yml | 47 ++++++++++++++-- tests/test_repository_workflow_interfaces.py | 57 ++++++++++++++------ 2 files changed, 86 insertions(+), 18 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 366d28ca2d..5e5fb9cb27 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,12 +12,10 @@ concurrency: cancel-in-progress: false jobs: - build: + checks: permissions: contents: read runs-on: ubuntu-latest - outputs: - artifact-id: ${{ steps.upload.outputs.artifact-id }} steps: # Tag rules and environment protection remain the release authorization boundary. @@ -63,6 +61,49 @@ jobs: UV_PYTHON="$python_version" make sync tests done make typecheck + + build: + # Test processes and filesystem mutations stay on the checks runner. + needs: checks + permissions: + contents: read + runs-on: ubuntu-latest + outputs: + artifact-id: ${{ steps.upload.outputs.artifact-id }} + + steps: + # Tag rules and environment protection remain the release authorization boundary. + - name: Checkout release validator from main + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: refs/heads/main + path: control + persist-credentials: false + sparse-checkout: .github/scripts + - name: Checkout release commit + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + ref: ${{ github.sha }} + path: release-source + fetch-depth: 0 + persist-credentials: false + - name: Setup Python for release validation + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 + with: + python-version: "3.14" + - name: Validate release provenance + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_SHA: ${{ github.sha }} + run: >- + python -I control/.github/scripts/verify_release.py + --repo release-source --tag "$RELEASE_TAG" --expected-sha "$RELEASE_SHA" + - name: Setup uv + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # setup-uv v9.0.0; uv 0.11.14 + with: + version: "0.11.14" + enable-cache: false + python-version: "3.14" - name: Build package working-directory: release-source run: uv build diff --git a/tests/test_repository_workflow_interfaces.py b/tests/test_repository_workflow_interfaces.py index 4032aef3ff..2807fb14da 100644 --- a/tests/test_repository_workflow_interfaces.py +++ b/tests/test_repository_workflow_interfaces.py @@ -176,8 +176,9 @@ def test_prospective_contract_preparation_removes_api_key_before_uv() -> None: assert recipe.index("unset OPENAI_API_KEY") < recipe.index("uv run") -def test_release_build_validates_before_executing_candidate_code() -> None: - build = _workflow_job("build", PUBLISH_WORKFLOW) +@pytest.mark.parametrize("job_name", ["checks", "build"]) +def test_release_build_validates_before_executing_candidate_code(job_name: str) -> None: + build = _workflow_job(job_name, PUBLISH_WORKFLOW) assert "contents: read" in build assert "id-token:" not in build @@ -187,8 +188,10 @@ def test_release_build_validates_before_executing_candidate_code() -> None: assert build.count("persist-credentials: false") == 2 assert "fetch-depth: 0" in build validation = build.index("python -I control/.github/scripts/verify_release.py") - assert validation < build.index('UV_PYTHON="$python_version" make sync tests') - assert build.index("make typecheck") < build.index("run: uv build") + candidate_command = ( + 'UV_PYTHON="$python_version" make sync tests' if job_name == "checks" else "run: uv build" + ) + assert validation < build.index(candidate_command) assert ' --tag "$RELEASE_TAG" --expected-sha "$RELEASE_SHA"' in build assert "enable-cache: false" in build @@ -199,20 +202,21 @@ def test_release_checks_fail_closed(tmp_path: Path, failed_check: str | None) -> if bash is None: pytest.skip("The publish workflow requires Bash.") workflow = yaml.safe_load(PUBLISH_WORKFLOW.read_text(encoding="utf-8")) - build = workflow["jobs"]["build"] - steps = build["steps"] + jobs = workflow["jobs"] + checks = jobs["checks"] + build = jobs["build"] + steps = checks["steps"] check_index = next(i for i, step in enumerate(steps) if step["name"] == "Check release source") check = steps[check_index] - package_index = next(i for i, step in enumerate(steps) if step.get("run") == "uv build") - assert check_index < package_index - assert ( - check["working-directory"] == steps[package_index]["working-directory"] == "release-source" - ) + package = next(step for step in build["steps"] if step.get("run") == "uv build") + assert check["working-directory"] == package["working-directory"] == "release-source" + assert build["needs"] == "checks" assert check["shell"] == "bash" assert check["env"] == {"OPENAI_API_KEY": "fake-for-tests", "UV_LOCKED": "1"} - assert "if" not in build and "continue-on-error" not in build - for step in steps[check_index:]: - assert "if" not in step and "continue-on-error" not in step + for job in (checks, build, jobs["publish"]): + assert "if" not in job and "continue-on-error" not in job + for step in job["steps"]: + assert "if" not in step and "continue-on-error" not in step bin_dir = tmp_path / "bin" bin_dir.mkdir() @@ -258,7 +262,7 @@ def test_release_checks_reject_stale_lockfile(tmp_path: Path) -> None: workflow = yaml.safe_load(PUBLISH_WORKFLOW.read_text(encoding="utf-8")) check = next( step - for step in workflow["jobs"]["build"]["steps"] + for step in workflow["jobs"]["checks"]["steps"] if step["name"] == "Check release source" ) dependency = tmp_path / "dependency" @@ -298,6 +302,29 @@ def test_release_checks_reject_stale_lockfile(tmp_path: Path) -> None: assert lockfile.read_bytes() == original_lock +def test_release_build_is_isolated_from_test_execution() -> None: + workflow = yaml.safe_load(PUBLISH_WORKFLOW.read_text(encoding="utf-8")) + checks = workflow["jobs"]["checks"] + build = workflow["jobs"]["build"] + # Separate GitHub-hosted jobs provide fresh runners, not just new directories. + assert checks["runs-on"] == build["runs-on"] == "ubuntu-latest" + assert checks["permissions"] == build["permissions"] == {"contents": "read"} + assert "outputs" not in checks + assert build["needs"] == "checks" + for job in (checks, build): + assert "env" not in job and "container" not in job + for step in job["steps"]: + action = step.get("uses", "") + assert not action.startswith(("actions/cache@", "actions/download-artifact@")) + if action.startswith("astral-sh/setup-uv@"): + assert step["with"]["enable-cache"] is False + if job is checks: + assert not action.startswith("actions/upload-artifact@") + build_commands = [step["run"] for step in build["steps"] if "run" in step] + assert len(build_commands) == 2 # Provenance validation, then packaging; no test execution. + assert build_commands[-1] == "uv build" + + def test_pypi_job_only_publishes_the_build_artifact() -> None: workflow = PUBLISH_WORKFLOW.read_text(encoding="utf-8") publish = _workflow_job("publish", PUBLISH_WORKFLOW)