From 68445cdf302078edcc8f535ffee6bd2e7c64462e Mon Sep 17 00:00:00 2001 From: Frank Bria <136862992+frankbria@users.noreply.github.com> Date: Tue, 27 Oct 2026 21:54:45 -0700 Subject: [PATCH] test(cleanroom): install proof9 from the published marketplace and run its tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cold-start harness has validated the CLI path end to end on every release and caught real defects, including a published 0.9.2 that could not run at all — found only because it installed from PyPI rather than from source. It never touched proof9: no Claude Code in the image, no reference to the plugin. Since the README's first screen now leads with the plugin install, the lead path was the one never tested cold. After the timed CLI walkthrough, and outside its total (that total is the figure the README quotes), the harness now runs the README's shell install, `claude plugin install proof9 --marketplace frankbria/codeframe`, then checks the installed copy is enabled, validates it, and runs its own test suite from there. CI tests the plugin in the repository; this tests what a stranger gets. Verification steps are marked documented=check so they do not read as docs drift; failures file P-PLUGIN-* findings. The image gains Claude Code on Node 22, pinned to the version CI tests the plugin against. tests/ci/test_cleanroom_plugin_path.py keeps the two pins equal and the install pointed at the published marketplace. Verified by building the image and running the plugin section of walkthrough.sh in it: install, enabled, validate and test all OK; 176 plugin tests pass from the installed copy. Full CI selection: 6043 passed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_019cEymaSpmZ1arjngxx1VNM --- scripts/quickstart-cleanroom/Dockerfile | 13 ++++++ scripts/quickstart-cleanroom/README.md | 16 ++++++-- scripts/quickstart-cleanroom/walkthrough.sh | 27 +++++++++++++ tests/ci/test_cleanroom_plugin_path.py | 44 +++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) create mode 100644 tests/ci/test_cleanroom_plugin_path.py diff --git a/scripts/quickstart-cleanroom/Dockerfile b/scripts/quickstart-cleanroom/Dockerfile index 40fb3784..d641a4a3 100644 --- a/scripts/quickstart-cleanroom/Dockerfile +++ b/scripts/quickstart-cleanroom/Dockerfile @@ -3,6 +3,9 @@ # Deliberately minimal: the issue's premise is "only Python 3.11+, uv, and an # ANTHROPIC_API_KEY". Nothing CodeFRAME-specific is baked in — installing it is # the first thing under test. git is here because `cf init` initializes a repo. +# +# Claude Code is here for the plugin path: the README's first screen installs +# proof9 into it, so a visitor taking that path already has it. FROM python:3.11-slim RUN apt-get update -qq \ @@ -12,6 +15,16 @@ RUN apt-get update -qq \ RUN curl -LsSf https://astral.sh/uv/install.sh | sh ENV PATH="/root/.local/bin:${PATH}" +# Claude Code, pinned to the version CI tests the plugin against +# (.github/workflows/test.yml); tests/ci/test_cleanroom_plugin_path.py keeps the +# two equal. The plugin itself is NOT baked in: installing it from the published +# marketplace is a step under test. +ARG CLAUDE_CODE_VERSION=2.1.301 +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - >/dev/null \ + && apt-get install -y -qq --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* \ + && npm install -g --silent "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" + # A first-time user has no git identity; `cf init`/`cf commit` need one. RUN git config --global user.email "cleanroom@example.com" \ && git config --global user.name "Cleanroom User" \ diff --git a/scripts/quickstart-cleanroom/README.md b/scripts/quickstart-cleanroom/README.md index a2a5937a..622d15b9 100644 --- a/scripts/quickstart-cleanroom/README.md +++ b/scripts/quickstart-cleanroom/README.md @@ -45,15 +45,25 @@ Artifacts land in `artifacts/` (gitignored) or the directory you name: documented command that misbehaves is recorded as a finding and the run continues, so one run collects every papercut instead of one per run. Steps the harness had to add because the docs omitted them are marked `documented=no` in - `timings.tsv` — that column is itself a docs-drift signal. + `timings.tsv` — that column is itself a docs-drift signal. `documented=check` + marks something else: a verification the harness runs on the published proof9 + plugin, not a step a user takes. +- **The plugin is installed from the published marketplace, not this checkout.** + After the timed CLI walkthrough, and outside its total, the harness runs the + README's shell install, `claude plugin install proof9 --marketplace + frankbria/codeframe`, then validates the installed copy and runs that copy's + own test suite. CI tests the plugin in the repository; this tests what a + stranger receives, for the same reason the CLI is installed from PyPI rather + than from source. - **`cf prd generate --brief-file brief.md` stands in for a human.** That command is a Socratic interview with AI-generated questions, so a canned answer list desynchronises on the first rejection (measured: 21 turns, 0 accepted). `--brief-file` answers the question actually asked from a fixed brief. This used to be a `responder.py` the harness carried itself; #1114 shipped the real thing, so the walkthrough now exercises a code path users have. -- The container carries only Python 3.11, `uv` and `git` — `cf` installs itself - into its own uv-managed venv, which is the thing under test. +- The container carries Python 3.11, `uv`, `git` and, for the plugin path, + Claude Code on Node 22. Neither `cf` nor proof9 is baked in: each is installed + from its published source, which is the thing under test. ## Regenerating the demo document diff --git a/scripts/quickstart-cleanroom/walkthrough.sh b/scripts/quickstart-cleanroom/walkthrough.sh index 4f1ee7ed..3a873eb2 100755 --- a/scripts/quickstart-cleanroom/walkthrough.sh +++ b/scripts/quickstart-cleanroom/walkthrough.sh @@ -239,6 +239,33 @@ TOTAL=$((RUN_END - RUN_START)) printf '\n\n===== TOTAL WALL CLOCK: %ss (%s min) =====\n' "$TOTAL" "$((TOTAL / 60))" printf 'TOTAL_SECONDS=%s\n' "$TOTAL" >"$ART/total.txt" +######################################## +# The proof9 plugin, installed the way a visitor to the README's first screen +# would: from the PUBLISHED marketplace on GitHub, not from this checkout. +# +# Outside the timed total above on purpose. That total is the "empty directory +# to a green cf proof run" figure the README quotes; this is a separate path. +# +# Same principle as installing codeframe-ai from PyPI rather than from source, +# which is how this harness caught a published 0.9.2 that could not run at all: +# CI tests the plugin in the repository, this tests the copy a stranger gets. +# Steps marked documented=check are verifications the harness runs on that +# copy, not steps a user takes. +######################################## +step "plugin-install" yes 300 -- claude plugin install proof9 --marketplace frankbria/codeframe +PLUGIN_DIR=$(node -e 'const d=require(process.env.HOME+"/.claude/plugins/installed_plugins.json");console.log(d.plugins["proof9@codeframe"][0].installPath)' 2>/dev/null) +if [ -z "$PLUGIN_DIR" ] || [ ! -d "$PLUGIN_DIR" ]; then + note P-PLUGIN-NOT-INSTALLED critical "claude plugin install left no install path for proof9@codeframe; the README's first-screen install does not produce a usable plugin." +else + printf '\n### INSTALLED PLUGIN: %s\n' "$PLUGIN_DIR" + step "plugin-enabled" check 60 -- bash -c "claude plugin list | grep -A4 'proof9@codeframe' | grep -q enabled" || + note P-PLUGIN-DISABLED critical "proof9 installed from the marketplace but is not enabled." + step "plugin-validate" check 120 -- claude plugin validate "$PLUGIN_DIR" || + note P-PLUGIN-INVALID critical "The published proof9 plugin fails claude plugin validate." + step "plugin-test" check 600 -- bash -c "cd '$PLUGIN_DIR' && claude plugin test ." || + note P-PLUGIN-TESTS critical "The published proof9 plugin's own tests fail when run from the installed copy." +fi + printf '\n===== TIMINGS =====\n' cat "$TIMINGS" printf '\n===== FINDINGS =====\n' diff --git a/tests/ci/test_cleanroom_plugin_path.py b/tests/ci/test_cleanroom_plugin_path.py new file mode 100644 index 00000000..30d5ee72 --- /dev/null +++ b/tests/ci/test_cleanroom_plugin_path.py @@ -0,0 +1,44 @@ +"""The cold-start harness checks the proof9 plugin a stranger actually receives. + +scripts/quickstart-cleanroom/ installs proof9 from the published marketplace and +runs its own tests from the installed copy. That only means something if it is +the published copy, and if the harness runs the same Claude Code that CI tests +the plugin against; otherwise the two disagree about which runtime the plugin +was checked on. +""" + +import re +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.v2 + +ROOT = Path(__file__).resolve().parents[2] +HARNESS = ROOT / "scripts" / "quickstart-cleanroom" + + +def test_cleanroom_pins_the_claude_code_ci_tests_the_plugin_against(): + workflows = "\n".join(p.read_text() for p in (ROOT / ".github" / "workflows").glob("*.yml")) + ci = set(re.findall(r"@anthropic-ai/claude-code@([0-9][0-9.]*)", workflows)) + assert len(ci) == 1, f"CI should pin exactly one Claude Code version, found {sorted(ci)}" + + dockerfile = (HARNESS / "Dockerfile").read_text() + pinned = re.search(r"^ARG CLAUDE_CODE_VERSION=([0-9][0-9.]*)$", dockerfile, re.MULTILINE) + assert pinned, "the cleanroom Dockerfile no longer pins CLAUDE_CODE_VERSION" + assert pinned.group(1) == next(iter(ci)), ( + f"cleanroom runs Claude Code {pinned.group(1)} but CI tests the plugin on " + f"{next(iter(ci))}" + ) + + +def test_cleanroom_installs_the_published_plugin_not_this_checkout(): + script = (HARNESS / "walkthrough.sh").read_text() + assert "claude plugin install proof9 --marketplace frankbria/codeframe" in script, ( + "the harness must install proof9 the way the README tells a visitor to" + ) + for check in ("plugin-validate", "plugin-test"): + assert f'step "{check}"' in script, f"the harness no longer runs {check} on the installed copy" + assert "@anthropic-ai/claude-code" not in script and "proof9/" not in script, ( + "the walkthrough must not install the plugin from a local path" + )