From 3958fc02a19a6e815e6c962175b50ffc410595e5 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Wed, 30 Sep 2026 00:08:21 +0900 Subject: [PATCH] fix(ci): sweep stale root-owned coverage workspaces on the OpenCode runner The coverage sandbox writes $RUNNER_TEMP/opencode-coverage-- as root, so the runner's per-job temp cleanup cannot remove them; about a dozen accumulated on cwlab-s1-04 again after the 2026-09-29 cleanup. #2491 removes the current run's workspace at job end, but runs dispatched before it, or jobs that never reach cleanup, leave theirs behind. The "Reclaim stale coverage images" step now also removes, with the sudo the job already uses, only run-numbered opencode-coverage-- and opencode-coverage-tool-build-- directories older than two hours, never this run's and never shared directories (artifact, source, sandbox-result, tool-build). Failure warns and does not block measurement. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01PtgtJk1WjqieqvLDb1w8uW --- .../workflows/opencode-review-dispatch.yml | 14 +++++++++ .../20260930-coverage-workspace-reclaim.md | 6 ++++ .../test_opencode_coverage_runner_hygiene.py | 30 +++++++++++++++++++ ...t_pr_review_autofix_nvidia_nim_contract.py | 2 +- 4 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.d/20260930-coverage-workspace-reclaim.md diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index 6870be9384..26308e1a58 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -611,6 +611,20 @@ jobs: if ! reclaim; then echo "::warning::Could not reclaim stale coverage images; continuing." fi + # The sandbox writes workspaces as root, so the runner's own temp + # cleanup cannot remove them; sweep ones older than two hours that + # a cancelled or pre-cleanup run left behind, never this run's. + if [ -d "${RUNNER_TEMP:-}" ]; then + # Only run-numbered directories; shared ones (artifact, source, + # sandbox-result, tool-build) are never swept. + if ! find "$RUNNER_TEMP" -mindepth 1 -maxdepth 1 -type d \ + \( -name 'opencode-coverage-[0-9]*-[0-9]*' -o -name 'opencode-coverage-tool-build-[0-9]*-[0-9]*' \) \ + -mmin +120 ! -name "opencode-coverage-${GITHUB_RUN_ID}-*" \ + ! -name "opencode-coverage-tool-build-${GITHUB_RUN_ID}-*" -print0 | + xargs -0 -r sudo rm -rf --; then + echo "::warning::Could not reclaim stale coverage workspaces; continuing." + fi + fi - name: Expose runner Rust toolchain # Self-hosted runners keep rustup's cargo in ~/.cargo/bin, which the diff --git a/CHANGELOG.d/20260930-coverage-workspace-reclaim.md b/CHANGELOG.d/20260930-coverage-workspace-reclaim.md new file mode 100644 index 0000000000..4e99d05c99 --- /dev/null +++ b/CHANGELOG.d/20260930-coverage-workspace-reclaim.md @@ -0,0 +1,6 @@ +## Fixed + +- Coverage jobs also remove run-numbered coverage workspaces older than two + hours. The sandbox writes them as root, so the runner's own temp cleanup + could not, and they refilled the self-hosted runner's disk. Shared coverage + directories are never removed. diff --git a/tests/test_opencode_coverage_runner_hygiene.py b/tests/test_opencode_coverage_runner_hygiene.py index a1c1f74dcd..a8910eac79 100644 --- a/tests/test_opencode_coverage_runner_hygiene.py +++ b/tests/test_opencode_coverage_runner_hygiene.py @@ -95,3 +95,33 @@ def test_missing_cargo_warns_readably(tmp_path: Path) -> None: result = _run(_step("Expose runner Rust toolchain")["run"], tmp_path, home=tmp_path / "home", path_dirs=[]) assert result.returncode == 0 assert "::warning::cargo is not installed" in result.stdout + + +def test_reclaim_removes_only_old_coverage_workspaces(tmp_path: Path) -> None: + """Root-owned sandbox workspaces left by earlier jobs are swept with sudo.""" + fake_bin = tmp_path / "bin" + _fake(fake_bin, "docker", "exit 0\n") + _fake(fake_bin, "sudo", 'exec "$@"\n') + runner_temp = tmp_path / "runner_temp" + old = runner_temp / "opencode-coverage-111-1" + old_build = runner_temp / "opencode-coverage-tool-build-111-1" + fresh = runner_temp / "opencode-coverage-222-1" + current = runner_temp / "opencode-coverage-333-1" + unrelated = runner_temp / "other-old" + shared = [runner_temp / f"opencode-coverage-{n}" for n in ("artifact", "sandbox-result", "source", "tool-build")] + for d in (old, old_build, fresh, current, unrelated, *shared): + (d / "x").mkdir(parents=True) + for d in (old, old_build, current, unrelated, *shared): + os.utime(d, (0, 0)) + script = _step("Reclaim stale coverage images")["run"] + result = subprocess.run( + ["bash", "-c", script], + env={"PATH": f"{fake_bin}:/usr/bin:/bin", "RUNNER_TEMP": str(runner_temp), + "GITHUB_RUN_ID": "333", "HOME": str(tmp_path)}, + capture_output=True, text=True, + ) + assert result.returncode == 0, result.stderr + assert not old.exists() and not old_build.exists() + assert fresh.exists() and current.exists() and unrelated.exists() + # Shared, non-run directories are never swept, however old. + assert all(d.exists() for d in shared) diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 5b045089d5..6e1918ed42 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "6870be9384a5e6abfea0074ad310c58329feec2a" +REVIEW_DISPATCH_BLOB_SHA = "26308e1a58f37f8aa15b3dd6d0453002ea1f50e8" def _workflow_text(path: Path) -> str: