diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 3909ea3b9c..62afd062d3 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -595,6 +595,37 @@ jobs: exit 1 fi + - name: Reclaim stale coverage images + # One OpenCode job runs per runner at a time, so coverage images older + # than two hours belong to finished or abandoned runs. Runs dispatched + # before per-run cleanup existed left them behind and filled the + # self-hosted guest disk. Failure here never blocks measurement. + run: | + set -uo pipefail + reclaim() { + docker image ls --filter reference=opencode-coverage-tools --filter until=2h --format '{{.ID}}' | + sort -u | while read -r image; do docker image rm -f "$image" || return 1; done && + docker image prune -f && + docker builder prune -f --filter until=24h + } + if ! reclaim; then + echo "::warning::Could not reclaim stale coverage images; continuing." + fi + + - name: Expose runner Rust toolchain + # Self-hosted runners keep rustup's cargo in ~/.cargo/bin, which the + # runner service does not put on PATH. Hosted images already have it. + run: | + set -euo pipefail + if command -v cargo >/dev/null 2>&1; then + exit 0 + fi + if [ -x "$HOME/.cargo/bin/cargo" ]; then + echo "$HOME/.cargo/bin" >>"$GITHUB_PATH" + exit 0 + fi + echo "::warning::cargo is not installed on this runner; Rust coverage will report a missing toolchain." + - name: Measure test and docstring evidence id: measure env: diff --git a/CHANGELOG.d/20260929-coverage-runner-hygiene.md b/CHANGELOG.d/20260929-coverage-runner-hygiene.md new file mode 100644 index 0000000000..1c37aeee44 --- /dev/null +++ b/CHANGELOG.d/20260929-coverage-runner-hygiene.md @@ -0,0 +1,8 @@ +## Fixed + +- Rust coverage on the self-hosted OpenCode runner can find `cargo` again. The + runner keeps it in `~/.cargo/bin`, which its service does not put on PATH, + so every Rust coverage gate failed and no fast-mlsirm review could approve. + A missing toolchain is now reported as such instead of a bare exception name. +- Coverage jobs remove coverage images older than two hours before measuring. + Runs dispatched before per-run cleanup existed had filled the runner disk. diff --git a/scripts/ci/materialize_base_rust_dependencies.py b/scripts/ci/materialize_base_rust_dependencies.py index 4a46f232c8..f56871a083 100644 --- a/scripts/ci/materialize_base_rust_dependencies.py +++ b/scripts/ci/materialize_base_rust_dependencies.py @@ -446,6 +446,12 @@ def _lock_for(root: str) -> str: vendor_dir = output_dir / "vendor" try: completed = _run_cargo_vendor(manifest_path, vendor_dir, sync_manifests) + except FileNotFoundError as exc: + raise RuntimeError( + f"could not run trusted cargo vendor for base manifest {lock_path}: " + "cargo is not installed or not on PATH on this runner " + "(self-hosted runners keep it in ~/.cargo/bin)" + ) from exc except (OSError, subprocess.TimeoutExpired) as exc: raise RuntimeError( f"could not run trusted cargo vendor for base manifest {lock_path}: " diff --git a/tests/test_materialize_base_rust_dependencies.py b/tests/test_materialize_base_rust_dependencies.py index b912c4ea4c..d8b409d342 100644 --- a/tests/test_materialize_base_rust_dependencies.py +++ b/tests/test_materialize_base_rust_dependencies.py @@ -540,3 +540,33 @@ def test_materialize_rejects_bad_sha_and_symlinked_output_dir(tmp_path: Path) -> linked_output.symlink_to(real_dir) with pytest.raises(ValueError, match="must not be a symlink"): materializer.materialize(Path("/unused"), "a" * 40, linked_output) + + +def test_missing_cargo_is_reported_as_a_runner_toolchain_gap( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str] +) -> None: + """A runner without cargo on PATH gets an actionable message, not a bare exception name.""" + repo = tmp_path / "repo" + _init_repo(repo) + (repo / "Cargo.toml").write_text( + '[package]\nname = "foo"\nversion = "0.1.0"\nedition = "2021"\n', encoding="utf-8" + ) + (repo / "src").mkdir() + (repo / "src" / "lib.rs").write_text("pub fn x() {}\n", encoding="utf-8") + (repo / "Cargo.lock").write_text( + 'version = 3\n\n[[package]]\nname = "foo"\nversion = "0.1.0"\n', encoding="utf-8" + ) + base_sha = _commit_all(repo) + + def no_cargo(*_args, **_kwargs): + raise FileNotFoundError(2, "No such file or directory", "cargo") + + monkeypatch.setattr(materializer, "_run_cargo_vendor", no_cargo) + exit_code = materializer.main( + ["--repo-root", str(repo), "--base-sha", base_sha, "--output-dir", str(tmp_path / "out")] + ) + + assert exit_code == 1 + err = capsys.readouterr().err + assert "cargo is not installed or not on PATH" in err + assert "~/.cargo/bin" in err diff --git a/tests/test_opencode_coverage_runner_hygiene.py b/tests/test_opencode_coverage_runner_hygiene.py new file mode 100644 index 0000000000..a1c1f74dcd --- /dev/null +++ b/tests/test_opencode_coverage_runner_hygiene.py @@ -0,0 +1,97 @@ +"""Keep the self-hosted OpenCode coverage runner usable between jobs. + +On 2026-09-29 `cwlab-s1-04` filled its disk twice with per-job coverage images, +and every Rust coverage run failed because `cargo` lives in `~/.cargo/bin`, +which the runner service does not put on PATH. +""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = REPO_ROOT / ".github/workflows/opencode-review-dispatch.yml" + + +def _steps() -> list[dict]: + return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"]["coverage-evidence"]["steps"] + + +def _step(name: str) -> dict: + return next(s for s in _steps() if s.get("name") == name) + + +def _index(name: str) -> int: + return next(i for i, s in enumerate(_steps()) if s.get("name") == name) + + +def _run(script: str, tmp_path: Path, *, home: Path, path_dirs: list[Path]) -> subprocess.CompletedProcess: + github_path = tmp_path / "github_path" + github_path.touch() + env = { + "HOME": str(home), + "PATH": os.pathsep.join([*map(str, path_dirs), "/usr/bin", "/bin"]), + "GITHUB_PATH": str(github_path), + } + return subprocess.run(["bash", "-c", script], env=env, capture_output=True, text=True) + + +def _fake(bin_dir: Path, name: str, body: str) -> None: + bin_dir.mkdir(parents=True, exist_ok=True) + tool = bin_dir / name + tool.write_text("#!/bin/sh\n" + body, encoding="utf-8") + tool.chmod(0o755) + + +def test_both_steps_run_before_coverage_measurement() -> None: + measure = _index("Measure test and docstring evidence") + assert _index("Reclaim stale coverage images") < measure + assert _index("Expose runner Rust toolchain") < measure + + +def test_image_reclaim_targets_only_old_coverage_images(tmp_path: Path) -> None: + calls = tmp_path / "calls" + fake_bin = tmp_path / "bin" + _fake(fake_bin, "docker", f'echo "$*" >> {calls}\n[ "$1 $2" = "image ls" ] && echo abc123\nexit 0\n') + result = _run(_step("Reclaim stale coverage images")["run"], tmp_path, home=tmp_path, path_dirs=[fake_bin]) + assert result.returncode == 0, result.stderr + log = calls.read_text(encoding="utf-8").splitlines() + assert "image ls --filter reference=opencode-coverage-tools --filter until=2h --format {{.ID}}" in log + assert "image rm -f abc123" in log + assert "image prune -f" in log + assert "builder prune -f --filter until=24h" in log + assert not any("prune -a" in line or "system prune" in line for line in log) + + +def test_image_reclaim_failure_warns_without_blocking_coverage(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + _fake(fake_bin, "docker", "exit 1\n") + result = _run(_step("Reclaim stale coverage images")["run"], tmp_path, home=tmp_path, path_dirs=[fake_bin]) + assert result.returncode == 0 + assert "::warning::" in result.stdout + + +def test_home_cargo_is_added_to_github_path(tmp_path: Path) -> None: + home = tmp_path / "home" + _fake(home / ".cargo" / "bin", "cargo", "exit 0\n") + result = _run(_step("Expose runner Rust toolchain")["run"], tmp_path, home=home, path_dirs=[]) + assert result.returncode == 0, result.stderr + assert (tmp_path / "github_path").read_text(encoding="utf-8") == f"{home}/.cargo/bin\n" + + +def test_cargo_already_on_path_is_left_alone(tmp_path: Path) -> None: + fake_bin = tmp_path / "bin" + _fake(fake_bin, "cargo", "exit 0\n") + result = _run(_step("Expose runner Rust toolchain")["run"], tmp_path, home=tmp_path / "home", path_dirs=[fake_bin]) + assert result.returncode == 0 + assert (tmp_path / "github_path").read_text(encoding="utf-8") == "" + + +def test_missing_cargo_warns_readably(tmp_path: Path) -> None: + result = _run(_step("Expose runner Rust toolchain")["run"], tmp_path, home=tmp_path / "home", path_dirs=[]) + assert result.returncode == 0 + assert "::warning::cargo is not installed" in result.stdout diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 3c0594c6aa..a24da5731e 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "3909ea3b9c285ba3dac09e0cccb6618fdb9adad6" +REVIEW_DISPATCH_BLOB_SHA = "62afd062d3bc8571fb0d350572d80ce81a9ae048" def _workflow_text(path: Path) -> str: