diff --git a/scripts/quickstart-cleanroom/Dockerfile b/scripts/quickstart-cleanroom/Dockerfile index 40fb3784..d641a4a3 100644 --- a/scripts/quickstart-cleanroom/Dockerfile +++ b/scripts/quickstart-cleanroom/Dockerfile @@ -3,6 +3,9 @@ # Deliberately minimal: the issue's premise is "only Python 3.11+, uv, and an # ANTHROPIC_API_KEY". Nothing CodeFRAME-specific is baked in — installing it is # the first thing under test. git is here because `cf init` initializes a repo. +# +# Claude Code is here for the plugin path: the README's first screen installs +# proof9 into it, so a visitor taking that path already has it. FROM python:3.11-slim RUN apt-get update -qq \ @@ -12,6 +15,16 @@ RUN apt-get update -qq \ RUN curl -LsSf https://astral.sh/uv/install.sh | sh ENV PATH="/root/.local/bin:${PATH}" +# Claude Code, pinned to the version CI tests the plugin against +# (.github/workflows/test.yml); tests/ci/test_cleanroom_plugin_path.py keeps the +# two equal. The plugin itself is NOT baked in: installing it from the published +# marketplace is a step under test. +ARG CLAUDE_CODE_VERSION=2.1.301 +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - >/dev/null \ + && apt-get install -y -qq --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* \ + && npm install -g --silent "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}" + # A first-time user has no git identity; `cf init`/`cf commit` need one. RUN git config --global user.email "cleanroom@example.com" \ && git config --global user.name "Cleanroom User" \ diff --git a/scripts/quickstart-cleanroom/README.md b/scripts/quickstart-cleanroom/README.md index a2a5937a..622d15b9 100644 --- a/scripts/quickstart-cleanroom/README.md +++ b/scripts/quickstart-cleanroom/README.md @@ -45,15 +45,25 @@ Artifacts land in `artifacts/` (gitignored) or the directory you name: documented command that misbehaves is recorded as a finding and the run continues, so one run collects every papercut instead of one per run. Steps the harness had to add because the docs omitted them are marked `documented=no` in - `timings.tsv` — that column is itself a docs-drift signal. + `timings.tsv` — that column is itself a docs-drift signal. `documented=check` + marks something else: a verification the harness runs on the published proof9 + plugin, not a step a user takes. +- **The plugin is installed from the published marketplace, not this checkout.** + After the timed CLI walkthrough, and outside its total, the harness runs the + README's shell install, `claude plugin install proof9 --marketplace + frankbria/codeframe`, then validates the installed copy and runs that copy's + own test suite. CI tests the plugin in the repository; this tests what a + stranger receives, for the same reason the CLI is installed from PyPI rather + than from source. - **`cf prd generate --brief-file brief.md` stands in for a human.** That command is a Socratic interview with AI-generated questions, so a canned answer list desynchronises on the first rejection (measured: 21 turns, 0 accepted). `--brief-file` answers the question actually asked from a fixed brief. This used to be a `responder.py` the harness carried itself; #1114 shipped the real thing, so the walkthrough now exercises a code path users have. -- The container carries only Python 3.11, `uv` and `git` — `cf` installs itself - into its own uv-managed venv, which is the thing under test. +- The container carries Python 3.11, `uv`, `git` and, for the plugin path, + Claude Code on Node 22. Neither `cf` nor proof9 is baked in: each is installed + from its published source, which is the thing under test. ## Regenerating the demo document diff --git a/scripts/quickstart-cleanroom/walkthrough.sh b/scripts/quickstart-cleanroom/walkthrough.sh index 4f1ee7ed..3a873eb2 100755 --- a/scripts/quickstart-cleanroom/walkthrough.sh +++ b/scripts/quickstart-cleanroom/walkthrough.sh @@ -239,6 +239,33 @@ TOTAL=$((RUN_END - RUN_START)) printf '\n\n===== TOTAL WALL CLOCK: %ss (%s min) =====\n' "$TOTAL" "$((TOTAL / 60))" printf 'TOTAL_SECONDS=%s\n' "$TOTAL" >"$ART/total.txt" +######################################## +# The proof9 plugin, installed the way a visitor to the README's first screen +# would: from the PUBLISHED marketplace on GitHub, not from this checkout. +# +# Outside the timed total above on purpose. That total is the "empty directory +# to a green cf proof run" figure the README quotes; this is a separate path. +# +# Same principle as installing codeframe-ai from PyPI rather than from source, +# which is how this harness caught a published 0.9.2 that could not run at all: +# CI tests the plugin in the repository, this tests the copy a stranger gets. +# Steps marked documented=check are verifications the harness runs on that +# copy, not steps a user takes. +######################################## +step "plugin-install" yes 300 -- claude plugin install proof9 --marketplace frankbria/codeframe +PLUGIN_DIR=$(node -e 'const d=require(process.env.HOME+"/.claude/plugins/installed_plugins.json");console.log(d.plugins["proof9@codeframe"][0].installPath)' 2>/dev/null) +if [ -z "$PLUGIN_DIR" ] || [ ! -d "$PLUGIN_DIR" ]; then + note P-PLUGIN-NOT-INSTALLED critical "claude plugin install left no install path for proof9@codeframe; the README's first-screen install does not produce a usable plugin." +else + printf '\n### INSTALLED PLUGIN: %s\n' "$PLUGIN_DIR" + step "plugin-enabled" check 60 -- bash -c "claude plugin list | grep -A4 'proof9@codeframe' | grep -q enabled" || + note P-PLUGIN-DISABLED critical "proof9 installed from the marketplace but is not enabled." + step "plugin-validate" check 120 -- claude plugin validate "$PLUGIN_DIR" || + note P-PLUGIN-INVALID critical "The published proof9 plugin fails claude plugin validate." + step "plugin-test" check 600 -- bash -c "cd '$PLUGIN_DIR' && claude plugin test ." || + note P-PLUGIN-TESTS critical "The published proof9 plugin's own tests fail when run from the installed copy." +fi + printf '\n===== TIMINGS =====\n' cat "$TIMINGS" printf '\n===== FINDINGS =====\n' diff --git a/tests/ci/test_cleanroom_plugin_path.py b/tests/ci/test_cleanroom_plugin_path.py new file mode 100644 index 00000000..30d5ee72 --- /dev/null +++ b/tests/ci/test_cleanroom_plugin_path.py @@ -0,0 +1,44 @@ +"""The cold-start harness checks the proof9 plugin a stranger actually receives. + +scripts/quickstart-cleanroom/ installs proof9 from the published marketplace and +runs its own tests from the installed copy. That only means something if it is +the published copy, and if the harness runs the same Claude Code that CI tests +the plugin against; otherwise the two disagree about which runtime the plugin +was checked on. +""" + +import re +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.v2 + +ROOT = Path(__file__).resolve().parents[2] +HARNESS = ROOT / "scripts" / "quickstart-cleanroom" + + +def test_cleanroom_pins_the_claude_code_ci_tests_the_plugin_against(): + workflows = "\n".join(p.read_text() for p in (ROOT / ".github" / "workflows").glob("*.yml")) + ci = set(re.findall(r"@anthropic-ai/claude-code@([0-9][0-9.]*)", workflows)) + assert len(ci) == 1, f"CI should pin exactly one Claude Code version, found {sorted(ci)}" + + dockerfile = (HARNESS / "Dockerfile").read_text() + pinned = re.search(r"^ARG CLAUDE_CODE_VERSION=([0-9][0-9.]*)$", dockerfile, re.MULTILINE) + assert pinned, "the cleanroom Dockerfile no longer pins CLAUDE_CODE_VERSION" + assert pinned.group(1) == next(iter(ci)), ( + f"cleanroom runs Claude Code {pinned.group(1)} but CI tests the plugin on " + f"{next(iter(ci))}" + ) + + +def test_cleanroom_installs_the_published_plugin_not_this_checkout(): + script = (HARNESS / "walkthrough.sh").read_text() + assert "claude plugin install proof9 --marketplace frankbria/codeframe" in script, ( + "the harness must install proof9 the way the README tells a visitor to" + ) + for check in ("plugin-validate", "plugin-test"): + assert f'step "{check}"' in script, f"the harness no longer runs {check} on the installed copy" + assert "@anthropic-ai/claude-code" not in script and "proof9/" not in script, ( + "the walkthrough must not install the plugin from a local path" + )