From 861449938ba80340bb3a98d57232d26f25b9527c Mon Sep 17 00:00:00 2001 From: umer Date: Fri, 30 Oct 2026 13:47:54 +0400 Subject: [PATCH 1/2] fallback judge default is Claude Sonnet 5.5, not Opus A make or refine whose configured judge fails now falls back to the Sonnet engine, so a run never spends Opus usage. Test and TESTING.md assertion updated to the shipped default. --- docs/TESTING.md | 2 +- src/judge-fallback.mjs | 4 ++-- test/critic.test.mjs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index 2c707e2..ce7abbe 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -74,7 +74,7 @@ first-draft-axi 2>&1 | tee -a evidence.txt; echo "exit ${PIPESTATU ``` - `NO_ENGINE`: sign in as the line says, then rerun. Nothing was written. -- `critique.judge_fallback` / a `help` line starting "The critic fell back": the first judge failed, cancelled, timed out or answered unreadably twice, so another signed-in judge (Claude Opus 5.5, then Codex) gave the verdict. The output names who gave it, which engine failed and its error, and says when the fallback shares an engine with the model that made the design (not independent). If every judge fails, that option gets `critique.code: "JUDGE_ALL_FAILED"` with each engine's error, is never marked passed, and the other options still finish. +- `critique.judge_fallback` / a `help` line starting "The critic fell back": the first judge failed, cancelled, timed out or answered unreadably twice, so another signed-in judge (Claude Sonnet 5.5, then Codex) gave the verdict. The output names who gave it, which engine failed and its error, and says when the fallback shares an engine with the model that made the design (not independent). If every judge fails, that option gets `critique.code: "JUDGE_ALL_FAILED"` with each engine's error, is never marked passed, and the other options still finish. - `JUDGE_FAILED … gave no usable … answer in 3 attempts`: the judging model's reply could not be read three times (the message lists each attempt's parse error and says if the output was cut off); no score and no pages are kept. Just rerun `make`. If a call succeeded only on a retry, the JSON says so in `critique.judge_retries`. Before this guide's last fix, `make` died this way 3 times in 7 runs, always while the plain baseline page was written as JSON inside JSON; the final walk passed first time. - `MAKE_FAILED … did not refine the design`: send `evidence.txt`; a refine on a normal `make` output should finish in minutes, not 14. - Stuck for more than 20 minutes with no output: Ctrl+C, then send `evidence.txt`. diff --git a/src/judge-fallback.mjs b/src/judge-fallback.mjs index 1a83307..2b30ae7 100644 --- a/src/judge-fallback.mjs +++ b/src/judge-fallback.mjs @@ -1,8 +1,8 @@ // A critic call that must not end the make: one more try on the same engine, then the next signed-in judge, every failure on record. import { engineOf, resolveEngine } from './byokit.mjs'; -// The judges tried after the configured one, in order: Claude Opus first, then Codex. Subscriptions only, never an API key. -export const FALLBACK_JUDGES = [{ agent: 'byokit:claude', model: 'claude-opus-5-5' }, { agent: 'byokit:codex', model: 'gpt-6.1-sol' }]; +// The judges tried after the configured one, in order: Claude Sonnet first, then Codex. Subscriptions only, never an API key. +export const FALLBACK_JUDGES = [{ agent: 'byokit:claude', model: 'claude-sonnet-5-5' }, { agent: 'byokit:codex', model: 'gpt-6.1-sol' }]; const same = (a, b) => a.agent === b.agent && a.model === b.model; const named = (j) => `${j.model} (${j.agent})`; diff --git a/test/critic.test.mjs b/test/critic.test.mjs index 0a4f8d7..28808dc 100644 --- a/test/critic.test.mjs +++ b/test/critic.test.mjs @@ -126,7 +126,7 @@ test('make does not die when every judge fails: that option is unscored and flag const { made, help } = await run('make', 'A notes app for Umer', '--out', out, '--options', '1'); const c = made.files[0].critique; assert.deepEqual([c.passed, c.score, c.code], [false, null, 'JUDGE_ALL_FAILED']); - assert.match(c.error, /every judge failed, so nothing was scored: gpt-6\.1-sol \(byokit:codex\):.*claude-opus-5-5 \(byokit:claude\):/s); + assert.match(c.error, /every judge failed, so nothing was scored: gpt-6\.1-sol \(byokit:codex\):.*claude-sonnet-5-5 \(byokit:claude\):/s); assert.ok(c.judge_failures.some((f) => f.agent === 'byokit:codex' && f.attempt === 2) && c.judge_failures.some((f) => f.agent === 'byokit:claude')); assert.match(readFileSync(out, 'utf8'), /Notes, version 1/); assert.ok(help.some((h) => /not scored because every judge failed/.test(h))); From 0906808625f8e0239a5e4228ab35257898ba60ce Mon Sep 17 00:00:00 2001 From: umer Date: Fri, 30 Oct 2026 15:09:38 +0400 Subject: [PATCH 2/2] journeys/critic.mjs imports its fixtures from test/fixtures The journey pointed at journeys/fixtures/critic-model.mjs and journeys/fixtures/judge-cli.mjs, which do not exist, so the critic journey could never run. --- journeys/critic.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/journeys/critic.mjs b/journeys/critic.mjs index aa81337..e154221 100644 --- a/journeys/critic.mjs +++ b/journeys/critic.mjs @@ -8,8 +8,8 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { promisify } from 'node:util'; import { renderCheck } from '../src/render-check.mjs'; -import { standIn } from './fixtures/critic-model.mjs'; -import { installJudge, referenceStandIn } from './fixtures/judge-cli.mjs'; +import { standIn } from '../test/fixtures/critic-model.mjs'; +import { installJudge, referenceStandIn } from '../test/fixtures/judge-cli.mjs'; const root = new URL('..', import.meta.url).pathname, out = process.env.DONE_DONE_EVIDENCE_ROOT ?? process.argv[2]; if (!out) throw Error('Give an evidence folder'); mkdirSync(out, { recursive: true });