test(cleanroom): install proof9 from the published marketplace and run its tests

scripts/quickstart-cleanroom/Dockerfile

@@ -3,6 +3,9 @@
 # Deliberately minimal: the issue's premise is "only Python 3.11+, uv, and an
 # ANTHROPIC_API_KEY". Nothing CodeFRAME-specific is baked in — installing it is
 # the first thing under test. git is here because `cf init` initializes a repo.
+#
+# Claude Code is here for the plugin path: the README's first screen installs
+# proof9 into it, so a visitor taking that path already has it.
 FROM python:3.11-slim
 
 RUN apt-get update -qq \
@@ -12,6 +15,16 @@ RUN apt-get update -qq \
 RUN curl -LsSf https://astral.sh/uv/install.sh | sh
 ENV PATH="/root/.local/bin:${PATH}"
 
+# Claude Code, pinned to the version CI tests the plugin against
+# (.github/workflows/test.yml); tests/ci/test_cleanroom_plugin_path.py keeps the
+# two equal. The plugin itself is NOT baked in: installing it from the published
+# marketplace is a step under test.
+ARG CLAUDE_CODE_VERSION=2.1.301
+RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - >/dev/null \
+    && apt-get install -y -qq --no-install-recommends nodejs \
+    && rm -rf /var/lib/apt/lists/* \
+    && npm install -g --silent "@anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}"
+
 # A first-time user has no git identity; `cf init`/`cf commit` need one.
 RUN git config --global user.email "cleanroom@example.com" \
     && git config --global user.name "Cleanroom User" \

scripts/quickstart-cleanroom/README.md

@@ -45,15 +45,25 @@ Artifacts land in `artifacts/` (gitignored) or the directory you name:
   documented command that misbehaves is recorded as a finding and the run
   continues, so one run collects every papercut instead of one per run. Steps the
   harness had to add because the docs omitted them are marked `documented=no` in
-  `timings.tsv` — that column is itself a docs-drift signal.
+  `timings.tsv` — that column is itself a docs-drift signal. `documented=check`
+  marks something else: a verification the harness runs on the published proof9
+  plugin, not a step a user takes.
+- **The plugin is installed from the published marketplace, not this checkout.**
+  After the timed CLI walkthrough, and outside its total, the harness runs the
+  README's shell install, `claude plugin install proof9 --marketplace
+  frankbria/codeframe`, then validates the installed copy and runs that copy's
+  own test suite. CI tests the plugin in the repository; this tests what a
+  stranger receives, for the same reason the CLI is installed from PyPI rather
+  than from source.
 - **`cf prd generate --brief-file brief.md` stands in for a human.** That command
   is a Socratic interview with AI-generated questions, so a canned answer list
   desynchronises on the first rejection (measured: 21 turns, 0 accepted).
   `--brief-file` answers the question actually asked from a fixed brief. This
   used to be a `responder.py` the harness carried itself; #1114 shipped the real
   thing, so the walkthrough now exercises a code path users have.
-- The container carries only Python 3.11, `uv` and `git` — `cf` installs itself
-  into its own uv-managed venv, which is the thing under test.
+- The container carries Python 3.11, `uv`, `git` and, for the plugin path,
+  Claude Code on Node 22. Neither `cf` nor proof9 is baked in: each is installed
+  from its published source, which is the thing under test.
 
 ## Regenerating the demo document
 

scripts/quickstart-cleanroom/walkthrough.sh

@@ -239,6 +239,33 @@ TOTAL=$((RUN_END - RUN_START))
 printf '\n\n===== TOTAL WALL CLOCK: %ss (%s min) =====\n' "$TOTAL" "$((TOTAL / 60))"
 printf 'TOTAL_SECONDS=%s\n' "$TOTAL" >"$ART/total.txt"
 
+########################################
+# The proof9 plugin, installed the way a visitor to the README's first screen
+# would: from the PUBLISHED marketplace on GitHub, not from this checkout.
+#
+# Outside the timed total above on purpose. That total is the "empty directory
+# to a green cf proof run" figure the README quotes; this is a separate path.
+#
+# Same principle as installing codeframe-ai from PyPI rather than from source,
+# which is how this harness caught a published 0.9.2 that could not run at all:
+# CI tests the plugin in the repository, this tests the copy a stranger gets.
+# Steps marked documented=check are verifications the harness runs on that
+# copy, not steps a user takes.
+########################################
+step "plugin-install" yes 300 -- claude plugin install proof9 --marketplace frankbria/codeframe
+PLUGIN_DIR=$(node -e 'const d=require(process.env.HOME+"/.claude/plugins/installed_plugins.json");console.log(d.plugins["proof9@codeframe"][0].installPath)' 2>/dev/null)
+if [ -z "$PLUGIN_DIR" ] || [ ! -d "$PLUGIN_DIR" ]; then
+  note P-PLUGIN-NOT-INSTALLED critical "claude plugin install left no install path for proof9@codeframe; the README's first-screen install does not produce a usable plugin."
+else
+  printf '\n### INSTALLED PLUGIN: %s\n' "$PLUGIN_DIR"
+  step "plugin-enabled" check 60 -- bash -c "claude plugin list | grep -A4 'proof9@codeframe' | grep -q enabled" ||
+    note P-PLUGIN-DISABLED critical "proof9 installed from the marketplace but is not enabled."
+  step "plugin-validate" check 120 -- claude plugin validate "$PLUGIN_DIR" ||
+    note P-PLUGIN-INVALID critical "The published proof9 plugin fails claude plugin validate."
+  step "plugin-test" check 600 -- bash -c "cd '$PLUGIN_DIR' && claude plugin test ." ||
+    note P-PLUGIN-TESTS critical "The published proof9 plugin's own tests fail when run from the installed copy."
+fi
+
 printf '\n===== TIMINGS =====\n'
 cat "$TIMINGS"
 printf '\n===== FINDINGS =====\n'

tests/ci/test_cleanroom_plugin_path.py

@@ -0,0 +1,44 @@
+"""The cold-start harness checks the proof9 plugin a stranger actually receives.
+
+scripts/quickstart-cleanroom/ installs proof9 from the published marketplace and
+runs its own tests from the installed copy. That only means something if it is
+the published copy, and if the harness runs the same Claude Code that CI tests
+the plugin against; otherwise the two disagree about which runtime the plugin
+was checked on.
+"""
+
+import re
+from pathlib import Path
+
+import pytest
+
+pytestmark = pytest.mark.v2
+
+ROOT = Path(__file__).resolve().parents[2]
+HARNESS = ROOT / "scripts" / "quickstart-cleanroom"
+
+
+def test_cleanroom_pins_the_claude_code_ci_tests_the_plugin_against():
+    workflows = "\n".join(p.read_text() for p in (ROOT / ".github" / "workflows").glob("*.yml"))
+    ci = set(re.findall(r"@anthropic-ai/claude-code@([0-9][0-9.]*)", workflows))
+    assert len(ci) == 1, f"CI should pin exactly one Claude Code version, found {sorted(ci)}"
+
+    dockerfile = (HARNESS / "Dockerfile").read_text()
+    pinned = re.search(r"^ARG CLAUDE_CODE_VERSION=([0-9][0-9.]*)$", dockerfile, re.MULTILINE)
+    assert pinned, "the cleanroom Dockerfile no longer pins CLAUDE_CODE_VERSION"
+    assert pinned.group(1) == next(iter(ci)), (
+        f"cleanroom runs Claude Code {pinned.group(1)} but CI tests the plugin on "
+        f"{next(iter(ci))}"
+    )
+
+
+def test_cleanroom_installs_the_published_plugin_not_this_checkout():
+    script = (HARNESS / "walkthrough.sh").read_text()
+    assert "claude plugin install proof9 --marketplace frankbria/codeframe" in script, (
+        "the harness must install proof9 the way the README tells a visitor to"
+    )
+    for check in ("plugin-validate", "plugin-test"):
+        assert f'step "{check}"' in script, f"the harness no longer runs {check} on the installed copy"
+    assert "@anthropic-ai/claude-code" not in script and "proof9/" not in script, (
+        "the walkthrough must not install the plugin from a local path"
+    )