From 0dceacc8ab726fe4bd85045aadfd475ffefb4a9a Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Fri, 15 May 2026 21:08:15 +0000 Subject: [PATCH 01/39] Fix cache savings formula; add cache + distillation summaries to reconcile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Cache savings — wrong baseline (fixes ~3.5x underestimate) Old formula in lib/resolve.py: avg_input_price = cost_val / (input_toks + output_toks) cache_savings = cache_read_toks * avg_input_price * 0.9 This was wrong on two counts: 1. cost_val is the POST-cache cost (already discounted by caching), so using it as the per-token-input-rate baseline drags the rate down to what was paid, not what the no-cache baseline would have cost. 2. Dividing by (input + output) mixes output tokens into the denominator, diluting the per-token rate further. For a real Sonnet run (497K cache reads, 50K fresh, 11K output, $0.46 total): old formula reported ~$0.38 saved. True savings vs no-cache baseline: $1.34. The new formula uses LiteLLM's list pricing (input_cost_per_token, cache_read_input_token_cost, cache_creation_input_token_cost) directly: read_savings = cache_read * (list_input_rate - cache_read_rate) write_overhead = cache_write * (cache_write_rate - list_input_rate) net = read_savings - write_overhead Bug only existed in lib/resolve.py — other loops didn't compute savings. ## Distillation savings — overstated by ~10x for long runs Old formula: cost_saved = total_tokens_saved * input_cost_per_token # where total_tokens_saved = (pre - post) * iterations This treated every iteration as if those tokens would have cost the full input rate. But after iteration 1, the un-distilled context would have been a cache hit at 0.1x. New formula prices iter 1 at full rate and iters 2..N at the cache-read rate. ## Status-log additions - lib/reconcile.py now tracks cache_read_tokens / cache_creation_tokens (was: no cache tracking at all) and includes the cache summary in its status log comment. - lib/resolve.py distillation summary is now built at status-log-post time via the new helper, instead of from the dead-code stub left behind in an earlier refactor. ## Shared helpers New in lib/formatting.py: - _model_rates(model) — pulls list + cache rates from LiteLLM with sensible fallbacks (0.1x / 1.25x) when the cache rates are missing. - build_cache_savings_summary(usage_path, model) - build_distillation_summary(pre, post, iters, model) Both helpers use plain round() instead of math.ceil() (the old code's ceil-of-float-product produced $0.25 from a true $0.24 due to fp imprecision). ## Tests Adds tests/test_cache_distill_savings.py with 16 test cases covering zero-cache, reads-only, writes-only, mixed reads/writes, write overhead swamping reads, single-iter distillation, multi-iter distillation, missing-model fallback, and a regression check that the new formula doesn't overstate vs. the old naive one. Full suite: 647 passed (was 631; +16). --- lib/formatting.py | 106 +++++++++++++++ lib/reconcile.py | 20 ++- lib/resolve.py | 95 ++------------ tests/test_cache_distill_savings.py | 191 ++++++++++++++++++++++++++++ 4 files changed, 324 insertions(+), 88 deletions(-) create mode 100644 tests/test_cache_distill_savings.py diff --git a/lib/formatting.py b/lib/formatting.py index cbbdf24..b7a2e6d 100644 --- a/lib/formatting.py +++ b/lib/formatting.py @@ -65,3 +65,109 @@ def _fmt_info(text) -> 'str | None': '| Metric | Value |', '|--------|-------|', ] + + +def _model_rates(model: str): + """Return (list_input_rate, cache_read_rate, cache_write_rate) for the model, + or (0, 0, 0) if LiteLLM doesn't know the model. Rates are per-token dollars. + + Falls back to standard Anthropic ratios (read = 0.1× input, write = 1.25× input) + if LiteLLM has the input rate but not the cache rates. + """ + try: + import litellm + info = litellm.get_model_info(model) + except Exception: + return (0.0, 0.0, 0.0) + list_input = float(info.get("input_cost_per_token") or 0) + if not list_input: + return (0.0, 0.0, 0.0) + cache_read = float(info.get("cache_read_input_token_cost") or 0) or (list_input * 0.1) + cache_write = float(info.get("cache_creation_input_token_cost") or 0) or (list_input * 1.25) + return (list_input, cache_read, cache_write) + + +def build_cache_savings_summary(usage_path: str = "/tmp/llm_usage.json", + model: 'str | None' = None) -> str: + """Build a cache savings summary string for the agent status log. + + Reads cache_read_tokens / cache_creation_tokens from usage_path (written + by write_usage in resolve / reconcile) and estimates savings against the + model's list input rate from LiteLLM's pricing table — NOT against the + blended post-cache cost (which was the bug fixed in this commit: + the old formula divided actual_cost by total_tokens, producing a rate + that was already discounted by caching AND diluted by output tokens, + leading to estimates 3-4× too low). + + Returns '' if cache was not used or the file is missing. + """ + import json + + try: + with open(usage_path) as f: + d = json.load(f) + except Exception: + return "" + + cache_read = int(d.get("cache_read_tokens", 0) or 0) + cache_write = int(d.get("cache_creation_tokens", 0) or 0) + if cache_read == 0 and cache_write == 0: + return "" + + parts = [] + if cache_read > 0: + parts.append(f"{_fmt_tok(cache_read)} tokens read from cache") + if cache_write > 0: + parts.append(f"{_fmt_tok(cache_write)} tokens written to cache") + + if model: + list_input, read_rate, write_rate = _model_rates(model) + if list_input > 0: + # Reads: saved (list_input - read_rate) per token (default 0.9× list). + # Writes: cost extra (write_rate - list_input) per token (default 0.25× list). + read_savings = cache_read * (list_input - read_rate) + write_overhead = cache_write * (write_rate - list_input) + net = read_savings - write_overhead + if net > 0: + parts.append(f"~${round(net, 2):.2f} saved") + + return f"**Cache:** {', '.join(parts)}" + + +def build_distillation_summary(pre_tokens: int, post_tokens: int, + iterations: int, model: 'str | None' = None) -> str: + """Build a distillation savings summary for the agent status log. + + Distillation reduces tokens-per-iteration by (pre - post). After iteration 1, + those tokens sit in the prompt cache, so the per-iter saving from iter 2 + onward is at the full input rate minus the cache-read rate (i.e., they + would have been cached too without distillation). + + To avoid overstating savings, we apply the cache-read rate for iters 2..N + (the previous formula used the full input rate, overstating by ~10× for + long runs since the alternative — sending more tokens every iter — would + also have been cached). + """ + if pre_tokens <= 0 or post_tokens <= 0 or iterations <= 0: + return "" + tokens_per_iter = pre_tokens - post_tokens + if tokens_per_iter <= 0: + return "" + + summary = ( + f"**Distillation:** {_fmt_tok(pre_tokens)} → {_fmt_tok(post_tokens)} tokens " + f"({_fmt_tok(tokens_per_iter)} saved/iter × {iterations} iters" + ) + + if model: + list_input, read_rate, _ = _model_rates(model) + if list_input > 0: + # First iteration would have paid full rate; subsequent iters would + # have been cache hits at read_rate. Distillation lets us avoid + # sending these tokens at all. + cost_saved = tokens_per_iter * (list_input + max(iterations - 1, 0) * read_rate) + if cost_saved > 0: + summary += f" = ~${round(cost_saved, 2):.2f} saved" + + summary += ")" + return summary diff --git a/lib/reconcile.py b/lib/reconcile.py index 99b9ae0..7ba5b73 100644 --- a/lib/reconcile.py +++ b/lib/reconcile.py @@ -236,7 +236,8 @@ def write_status(success, explanation): json.dump({"success": success, "explanation": explanation}, f) -def write_usage(input_tokens, output_tokens, cost, iterations): +def write_usage(input_tokens, output_tokens, cost, iterations, + cache_read_tokens=0, cache_creation_tokens=0): """Write token usage to /tmp/llm_usage.json.""" with open("/tmp/llm_usage.json", "w") as f: json.dump( @@ -245,6 +246,8 @@ def write_usage(input_tokens, output_tokens, cost, iterations): "output_tokens": output_tokens, "cost": cost, "iterations": iterations, + "cache_read_tokens": cache_read_tokens, + "cache_creation_tokens": cache_creation_tokens, }, f, ) @@ -461,6 +464,8 @@ def main(): total_input_tokens = 0 total_output_tokens = 0 total_cost = 0.0 + total_cache_read_tokens = 0 + total_cache_creation_tokens = 0 finish_args = None last_iteration = 0 no_tool_call_count = 0 @@ -587,6 +592,10 @@ def main(): if usage: total_input_tokens += getattr(usage, "prompt_tokens", 0) total_output_tokens += getattr(usage, "completion_tokens", 0) + prompt_details = getattr(usage, "prompt_tokens_details", None) + if prompt_details: + total_cache_read_tokens += getattr(prompt_details, "cached_tokens", 0) or 0 + total_cache_creation_tokens += getattr(prompt_details, "cache_creation_input_tokens", 0) or 0 cost = getattr(response, "_hidden_params", {}).get("response_cost", None) if cost: total_cost += cost @@ -752,14 +761,19 @@ def _compaction_llm_call(prompt_messages, max_tokens): loop_completed_naturally = True finally: - write_usage(total_input_tokens, total_output_tokens, total_cost, last_iteration + 1) + write_usage(total_input_tokens, total_output_tokens, total_cost, last_iteration + 1, + cache_read_tokens=total_cache_read_tokens, + cache_creation_tokens=total_cache_creation_tokens) # Post status log to the PR if collected. if status_log and PR_NUMBER and GITHUB_REPO: try: + from lib.formatting import build_cache_savings_summary log_text = "\n\n".join(f"**Iter {i}:** {text}" for i, text in status_log) model_header = f"\U0001f916 **Model:** `{ALIAS}` (`{LLM_MODEL}`)\n\n" if ALIAS else "" - comment_body = f"## Agent Status Log\n\n{model_header}{log_text}" + cache_summary = build_cache_savings_summary(model=LLM_MODEL) + header_block = cache_summary + "\n\n" if cache_summary else "" + comment_body = f"## Agent Status Log\n\n{model_header}{header_block}{log_text}" comment_file = "/tmp/rdb_status_log_comment.txt" with open(comment_file, "w") as f: f.write(comment_body) diff --git a/lib/resolve.py b/lib/resolve.py index 273b9c4..8e9a71f 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -1035,47 +1035,6 @@ def build_cost_table(): lines = TABLE_HEADER[:] lines += [f"| {k} | {v} |" for k, v in rows] return "\n".join(lines) -def build_cache_savings_summary(): - """Build a cache savings summary string for the agent status log. - - Returns a non-empty string if cache was used, or '' otherwise. - """ - import math - - try: - with open("/tmp/llm_usage.json") as f: - d = json.load(f) - except Exception: - return "" - - input_toks = int(d.get("input_tokens", 0)) - output_toks = int(d.get("output_tokens", 0)) - cost_val = float(d.get("cost") or 0) - cache_read_toks = int(d.get("cache_read_tokens", 0)) - cache_creation_toks = int(d.get("cache_creation_tokens", 0)) - - if cache_read_toks == 0 and cache_creation_toks == 0: - return "" - - from lib.formatting import _fmt_tok - - parts = [] - if cache_read_toks > 0: - parts.append(f"{_fmt_tok(cache_read_toks)} tokens read from cache") - if cache_creation_toks > 0: - parts.append(f"{_fmt_tok(cache_creation_toks)} tokens written to cache") - - # Estimate savings - uncached_input = input_toks - cache_read_toks - if cache_read_toks > 0 and uncached_input > 0 and (input_toks + output_toks) > 0: - avg_input_price = cost_val / (input_toks + output_toks) - cache_savings = cache_read_toks * avg_input_price * 0.9 - rounded_savings = math.ceil(cache_savings * 100) / 100 - parts.append(f"~${rounded_savings:.2f} saved") - - return f"**Cache:** {', '.join(parts)}" - - def write_status(success, explanation, no_op=False): """Write resolve status to /tmp/resolve_status.json.""" payload = {"success": success, "explanation": explanation} @@ -1188,7 +1147,6 @@ def main(): distill_cost = 0.0 pre_distill_tokens = 0 # token estimate before distillation post_distill_tokens = 0 # token estimate after distillation (0 = distillation did not run) - distillation_summary = "" if DISTILL_ENABLED: try: from lib.distill import maybe_distill @@ -1260,9 +1218,8 @@ def main(): last_iteration = 0 no_tool_call_count = 0 status_log = [] # list of (iteration, status_text) tuples - # Add distillation summary as first status log entry if distillation ran - if distillation_summary: - status_log.append((0, f"**Distillation summary:** {distillation_summary}")) + # Distillation summary (with savings) is added to the status log header + # at posting time — see the call to build_distillation_summary below. transient_error_counter = [0] # mutable counter for transient API errors rate_limit_retries = 0 @@ -1829,51 +1786,19 @@ def _compaction_llm_call(prompt_messages, max_tokens): cache_read_tokens=total_cache_read_tokens, cache_creation_tokens=total_cache_creation_tokens) - def _fmt_tokens(n): - """Format a token count as a human-readable string: '1.3 M', '73 K', '850'.""" - if n >= 1_000_000: - return f"{n / 1_000_000:.1f} M" - if n >= 1_000: - return f"{n / 1_000:.0f} K" - return str(n) - - # Compute distillation savings summary (if distillation ran) - distillation_summary = "" - if distillation_ran and pre_distill_tokens > 0 and post_distill_tokens > 0: - try: - actual_iterations = last_iteration + 1 - tokens_saved_per_iter = pre_distill_tokens - post_distill_tokens - total_tokens_saved = tokens_saved_per_iter * actual_iterations - # Estimate cost saved using LiteLLM model pricing - cost_saved_str = "" - try: - model_info = litellm.get_model_info(LLM_MODEL) - input_cost_per_token = model_info.get("input_cost_per_token", 0) - if input_cost_per_token: - cost_saved = total_tokens_saved * input_cost_per_token - cost_saved_str = f", ~${cost_saved:.2f} saved" - except Exception: - pass # cost estimate is optional; skip if unavailable - distillation_summary = ( - f"**Distillation:** {_fmt_tokens(pre_distill_tokens)} tokens → {_fmt_tokens(post_distill_tokens)} tokens " - f"({_fmt_tokens(tokens_saved_per_iter)} saved/iter × {actual_iterations} iters = " - f"{_fmt_tokens(total_tokens_saved)} tokens saved{cost_saved_str})" - ) - print(f"Distillation savings: {distillation_summary}") - except Exception as e: - print(f"Could not compute distillation savings: {e}") - # Post rolling status log as issue comment (if any entries were collected) if status_log and ISSUE_NUMBER and GITHUB_REPO: try: + from lib.formatting import build_cache_savings_summary, build_distillation_summary log_text = "\n\n".join(f"**Iter {i}:** {text}" for i, text in status_log) model_header = f"\U0001f916 **Model:** `{ALIAS}` (`{LLM_MODEL}`)\n\n" if ALIAS else "" - cache_summary = build_cache_savings_summary() - header_parts = [] - if distillation_summary: - header_parts.append(distillation_summary) - if cache_summary: - header_parts.append(cache_summary) + distillation_summary = "" + if distillation_ran: + distillation_summary = build_distillation_summary( + pre_distill_tokens, post_distill_tokens, last_iteration + 1, LLM_MODEL + ) + cache_summary = build_cache_savings_summary(model=LLM_MODEL) + header_parts = [p for p in (distillation_summary, cache_summary) if p] header_block = "\n\n".join(header_parts) if header_block: comment_body = f"## Agent Status Log\n\n{model_header}{header_block}\n\n{log_text}" diff --git a/tests/test_cache_distill_savings.py b/tests/test_cache_distill_savings.py new file mode 100644 index 0000000..6ef49ba --- /dev/null +++ b/tests/test_cache_distill_savings.py @@ -0,0 +1,191 @@ +"""Tests for cache and distillation savings helpers in lib/formatting.py. + +These helpers compute the savings strings shown in the agent status log. +The previous formula in resolve.py used `cost / (input + output)` as a proxy +for the input rate, which produced estimates 3-4× too low because: + - The numerator was the post-cache cost (already discounted) + - The denominator mixed output tokens with input tokens + +The new helpers use LiteLLM's list pricing (`input_cost_per_token` and the +explicit `cache_read_input_token_cost` / `cache_creation_input_token_cost`). +""" + +import json +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +REPO_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from lib.formatting import ( + build_cache_savings_summary, + build_distillation_summary, + _model_rates, +) + + +# Pricing fixture matching Claude Sonnet 4.6: $3/M input, $15/M output, +# $0.30/M cache reads, $3.75/M cache writes. +SONNET_PRICES = { + "input_cost_per_token": 3e-6, + "output_cost_per_token": 1.5e-5, + "cache_read_input_token_cost": 3e-7, + "cache_creation_input_token_cost": 3.75e-6, +} + + +def _write_usage(tmp_path, **kwargs): + """Write a /tmp-style usage JSON file and return its path.""" + p = tmp_path / "llm_usage.json" + p.write_text(json.dumps(kwargs)) + return str(p) + + +# --------------------------------------------------------------------------- +# _model_rates +# --------------------------------------------------------------------------- + +class TestModelRates: + def test_returns_zeros_for_unknown_model(self): + # The patched get_model_info raises; helper must return zeros. + with patch("litellm.get_model_info", side_effect=Exception("unknown")): + assert _model_rates("nonsense/model") == (0.0, 0.0, 0.0) + + def test_returns_explicit_rates(self): + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + inp, read, write = _model_rates("anthropic/claude-sonnet-4-6") + assert inp == 3e-6 + assert read == 3e-7 + assert write == 3.75e-6 + + def test_falls_back_to_standard_ratios_when_cache_rates_missing(self): + # Only input rate present — fallback to 0.1× / 1.25× + with patch("litellm.get_model_info", + return_value={"input_cost_per_token": 3e-6}): + inp, read, write = _model_rates("anthropic/claude-sonnet-4-6") + assert inp == 3e-6 + assert read == pytest.approx(3e-7) + assert write == pytest.approx(3.75e-6) + + +# --------------------------------------------------------------------------- +# build_cache_savings_summary +# --------------------------------------------------------------------------- + +class TestCacheSavingsSummary: + def test_empty_when_no_cache(self, tmp_path): + path = _write_usage(tmp_path, cache_read_tokens=0, cache_creation_tokens=0) + assert build_cache_savings_summary(path, "anthropic/claude-sonnet-4-6") == "" + + def test_empty_when_file_missing(self): + assert build_cache_savings_summary("/nonexistent/file.json", "any/model") == "" + + def test_reads_only_with_savings(self, tmp_path): + """497K cache reads on Sonnet → ~$1.34 saved (matches bridge-analysis run).""" + path = _write_usage(tmp_path, cache_read_tokens=497_000, cache_creation_tokens=0) + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_cache_savings_summary(path, "anthropic/claude-sonnet-4-6") + # 497_000 × (3e-6 - 3e-7) = 497_000 × 2.7e-6 = $1.3419 → round = $1.34 + assert "497K tokens read from cache" in out + assert "$1.34 saved" in out + + def test_no_model_omits_savings(self, tmp_path): + path = _write_usage(tmp_path, cache_read_tokens=497_000, cache_creation_tokens=0) + out = build_cache_savings_summary(path, model=None) + assert "497K tokens read from cache" in out + assert "saved" not in out + + def test_writes_only_no_savings_estimate(self, tmp_path): + path = _write_usage(tmp_path, cache_read_tokens=0, cache_creation_tokens=50_000) + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_cache_savings_summary(path, "anthropic/claude-sonnet-4-6") + # Writes alone are a NET OVERHEAD, not savings — no $ figure shown. + assert "50K tokens written to cache" in out + assert "saved" not in out + + def test_reads_and_writes_net_savings(self, tmp_path): + """Mixed reads and writes: read savings net out write overhead.""" + path = _write_usage(tmp_path, + cache_read_tokens=500_000, + cache_creation_tokens=100_000) + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_cache_savings_summary(path, "anthropic/claude-sonnet-4-6") + # read_savings = 500_000 × 2.7e-6 = $1.35 + # write_overhead = 100_000 × 0.75e-6 = $0.075 + # net = $1.275 → round = $1.27 (banker's rounding) or $1.28 + import re + m = re.search(r"\$([\d.]+) saved", out) + assert m is not None + assert float(m.group(1)) == pytest.approx(1.27, abs=0.02) + + def test_write_overhead_swamps_reads_no_savings(self, tmp_path): + """If write overhead > read savings, omit the savings line.""" + path = _write_usage(tmp_path, + cache_read_tokens=1_000, + cache_creation_tokens=10_000_000) + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_cache_savings_summary(path, "anthropic/claude-sonnet-4-6") + assert "saved" not in out + + +# --------------------------------------------------------------------------- +# build_distillation_summary +# --------------------------------------------------------------------------- + +class TestDistillationSummary: + def test_empty_when_no_savings(self): + # post >= pre means distillation actually grew the context — bail. + assert build_distillation_summary(100, 200, 10, None) == "" + + def test_empty_when_zero_iterations(self): + assert build_distillation_summary(1000, 500, 0, None) == "" + + def test_basic_summary_no_model(self): + out = build_distillation_summary(100_000, 20_000, 15, None) + assert "100K" in out + assert "20K" in out + assert "80K saved/iter" in out + assert "15 iters" in out + assert "saved" not in out.split("saved/iter")[1] # no cost figure + + def test_cost_savings_uses_cache_aware_formula(self): + """The dominant cost saving is iter-1 (uncached) plus (N-1) cache hits. + + For 80K tokens × 15 iters on Sonnet: + iter 1: 80_000 × $3e-6 = $0.24 + iters 2..15 (14 iters cached): 80_000 × $3e-7 × 14 = $0.336 + total = $0.576 → round = $0.58 + """ + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_distillation_summary(100_000, 20_000, 15, + "anthropic/claude-sonnet-4-6") + assert "$0.58 saved" in out + + def test_single_iteration_pays_full_rate_only(self): + """For a 1-iter run, only the uncached first-iter cost applies.""" + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_distillation_summary(100_000, 20_000, 1, + "anthropic/claude-sonnet-4-6") + # 80_000 × $3e-6 = $0.24 + assert "$0.24 saved" in out + + def test_not_overstated_vs_naive_formula(self): + """The new formula must produce significantly lower savings than the + old (now-removed) formula which used full input rate every iter. + + Old: 80K × $3e-6 × 15 = $3.60 + New: $0.24 + 80K × $3e-7 × 14 = $0.576 + """ + with patch("litellm.get_model_info", return_value=SONNET_PRICES): + out = build_distillation_summary(100_000, 20_000, 15, + "anthropic/claude-sonnet-4-6") + # Naive (incorrect) formula would have produced ~$3.60 — assert we're + # well under that. + import re + m = re.search(r"\$([\d.]+) saved", out) + assert m is not None + assert float(m.group(1)) < 1.0 From 58459263f4bf01378390a4d18eedd2ec8aa2d846 Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Fri, 15 May 2026 21:47:50 +0000 Subject: [PATCH 02/39] Remove upper cap on distillation; let Tier 2 handle any repo size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridge-analysis at 340K tokens fell just over the 300K DISTILL_STRUCT_EXTRACT_LIMIT and skipped distillation entirely — even though the Tier 2 path (structural extract → identify relevant files → read those in full) would have worked fine for it. The structural extract is ~10-15% of total repo tokens (39K for bridge-analysis), so the LLM call inside Tier 2 sees a small input regardless of how big the underlying repo is. The 300K hard cap wasn't protecting anything meaningful for repos in the 300K-1M range. Removed it. Two tiers now: 1. <= 100K tokens (DISTILL_SMALL_REPO_LIMIT): send full codebase 2. otherwise: structural-extract path For very large codebases (~5-10M tokens, where the structural extract itself would overflow a 1M-token model context window) the outer try/except in maybe_distill catches the LLM error and returns the original un-distilled context as a graceful fallback. Added a test to lock in that behavior. Updated docstrings to describe the new two-tier structure, removed the unused DISTILL_STRUCT_EXTRACT_LIMIT constant, replaced the now- obsolete "skips at Tier 3" test with "uses Tier 2 path on larger repos", and added the explicit overflow-fallback test. --- lib/distill.py | 39 +++++++++------ tests/test_distill.py | 112 +++++++++++++++++++++++++++++------------- 2 files changed, 103 insertions(+), 48 deletions(-) diff --git a/lib/distill.py b/lib/distill.py index 8b82b63..e343022 100644 --- a/lib/distill.py +++ b/lib/distill.py @@ -3,11 +3,20 @@ Gathers the target codebase, makes a single LLM call to extract only what's relevant to the task, and returns a focused context block for the agent. -Three tiers: - 1. Small repos (<= DISTILL_SMALL_REPO_LIMIT tokens): send full codebase - 2. Medium repos (<= DISTILL_STRUCT_EXTRACT_LIMIT tokens): send structural - extract, identify relevant files, read those in full (second pass) - 3. Large repos: skip distillation entirely (return original context) +Two tiers: + 1. Small repos (<= DISTILL_SMALL_REPO_LIMIT tokens): send the full codebase + in one call. + 2. Larger repos: send a compact structural extract (function/class signatures + for Python, head-lines for other files), use a first LLM call to identify + relevant files, then read only those in full and pass them through a + second distillation call. + +There is no upper cap on repo size. The structural extract is roughly 10-15% +of the codebase by tokens (varies with file mix), so the Tier-2 LLM input +remains well below typical 1M-token model context windows for codebases up +to several million tokens. If the extract ever does overflow the model's +context window, the outer try/except returns the original (un-distilled) +context as a graceful fallback. """ import ast @@ -19,9 +28,9 @@ # --- Constants (all hardcoded — no user-facing config) --- -# Token budget for the distillation call input +# Token budget below which we skip the structural-extract step and just send +# the whole codebase to the distill LLM in one shot. DISTILL_SMALL_REPO_LIMIT = 100_000 # tokens — try to send whole codebase -DISTILL_STRUCT_EXTRACT_LIMIT = 300_000 # tokens — for medium repos, use structural extract # Per-file size caps before truncation (in characters) SOURCE_FILE_CAP = 50_000 # for source/config files @@ -454,9 +463,14 @@ def maybe_distill(repo_context, issue_context, model, root="."): total_output += out total_cost += cost - elif total_tokens <= DISTILL_STRUCT_EXTRACT_LIMIT: - # Tier 2: Medium repo — structural extract + second pass - print(" [Distill] Tier 2 (medium repo): structural extract + targeted read") + else: + # Tier 2: Larger repo — structural extract + second pass. + # No upper bound here: the structural extract is ~10-15% of total + # tokens, so this path scales to multi-million-token codebases + # before hitting model context limits. If the extract does + # overflow the model context, the outer try/except returns the + # un-distilled context as a graceful fallback. + print(f" [Distill] Tier 2 (~{total_tokens:,} tokens): structural extract + targeted read") # The distillation input uses the extract without line numbers # (line numbers aren't useful for the distillation LLM's task of # identifying relevant files). @@ -488,11 +502,6 @@ def maybe_distill(repo_context, issue_context, model, root="."): total_output += out total_cost += cost - else: - # Tier 3: Large repo — skip - print(f" [Distill] Repo too large (~{total_tokens:,} tokens) — skipping distillation") - return fallback - if not result or not result.strip(): print(" [Distill] Empty distillation result — falling back") return fallback diff --git a/tests/test_distill.py b/tests/test_distill.py index cf457d8..f390770 100644 --- a/tests/test_distill.py +++ b/tests/test_distill.py @@ -13,7 +13,6 @@ SOURCE_FILE_CAP, OTHER_FILE_CAP, DISTILL_SMALL_REPO_LIMIT, - DISTILL_STRUCT_EXTRACT_LIMIT, DISTILL_OUTPUT_TOKENS, TRUNC_HALF_CHARS, OTHER_TRUNC_HALF_CHARS, @@ -378,26 +377,50 @@ def test_small_repo_sends_full_codebase(self, tmp_path): user_msg = [m for m in messages if m["role"] == "user"][0]["content"] assert "" in user_msg - def test_large_repo_skips_distillation(self, tmp_path): - """Large repos (> DISTILL_STRUCT_EXTRACT_LIMIT) skip distillation.""" - files = { - "main.py": "print('hello')", - } + def test_large_repo_uses_structural_extract_path(self, tmp_path): + """Repos far above the small-repo limit go through the Tier 2 path + (structural extract + identify + targeted read) — there is no upper + cap that would skip distillation entirely.""" + files = {"main.py": "print('hello')"} make_git_repo(tmp_path, files) - - original_context = "## Repo context" - - # Mock gather_repo_files to return a huge file list + + # File list that estimates well above DISTILL_SMALL_REPO_LIMIT + # (would previously have been past the old 300K Tier-3 cutoff). huge_files = [ - {"path": f"file_{i}.py", "content": "x" * 100000, "is_source": True, "truncated": False} - for i in range(200) + {"path": f"file_{i}.py", "content": f"def fn_{i}(): pass\n", + "is_source": True, "truncated": False} + for i in range(50) ] - - with patch("lib.distill.gather_repo_files", return_value=huge_files): - result = maybe_distill(original_context, "Fix a bug", "anthropic/claude-sonnet-4-5", root=str(tmp_path)) - - assert result[0] == original_context # unchanged - assert result[1] == 0 + + identify_response = MagicMock() + identify_response.choices = [MagicMock()] + identify_response.choices[0].message.content = "file_1.py\nfile_2.py" + identify_response.usage = MagicMock() + identify_response.usage.prompt_tokens = 1000 + identify_response.usage.completion_tokens = 50 + identify_response._hidden_params = {"response_cost": 0.01} + + distill_response = MagicMock() + distill_response.choices = [MagicMock()] + distill_response.choices[0].message.content = "distilled output" + distill_response.usage = MagicMock() + distill_response.usage.prompt_tokens = 500 + distill_response.usage.completion_tokens = 100 + distill_response._hidden_params = {"response_cost": 0.02} + + # Token estimate is well above DISTILL_SMALL_REPO_LIMIT to force Tier 2. + with patch("lib.distill.gather_repo_files", return_value=huge_files), \ + patch("lib.distill.estimate_tokens_for_files", + return_value=DISTILL_SMALL_REPO_LIMIT * 10), \ + patch("lib.distill.completion", + side_effect=[identify_response, distill_response]) as mock_comp: + result = maybe_distill("## Repo context", "Fix a bug", + "anthropic/claude-sonnet-4-5", + root=str(tmp_path)) + + # Two LLM calls: identify + distill (the Tier 2 path), not skipped. + assert mock_comp.call_count == 2 + assert result[0] == "distilled output" def test_medium_repo_uses_structural_extract(self, tmp_path): """Medium repos use structural extract approach.""" @@ -484,7 +507,6 @@ def test_caps_are_positive(self): assert SOURCE_FILE_CAP > 0 assert OTHER_FILE_CAP > 0 assert DISTILL_SMALL_REPO_LIMIT > 0 - assert DISTILL_STRUCT_EXTRACT_LIMIT > DISTILL_SMALL_REPO_LIMIT assert DISTILL_OUTPUT_TOKENS > 0 @@ -751,30 +773,54 @@ def test_above_threshold_distill_called_and_result_returned(self, tmp_path): assert result[0] == "distilled result" assert result[0] != original_context - def test_below_threshold_distill_not_called_original_returned(self, tmp_path): - """Tier 3 (huge repo, > DISTILL_STRUCT_EXTRACT_LIMIT tokens): distill() NOT called. - - When the repo is very large (above DISTILL_STRUCT_EXTRACT_LIMIT), maybe_distill() - falls back without making any LLM call, and returns the original repo_context - unchanged with zero token counts. - """ + def test_empty_file_list_returns_original(self, tmp_path): + """When gather_repo_files returns nothing, maybe_distill bails without + any LLM call and the original context flows through unchanged. This is + the only remaining "skip distillation entirely" path — there is no + upper-size cap.""" files = {"placeholder.py": "x"} make_git_repo(tmp_path, files) - # Token estimate clearly above DISTILL_STRUCT_EXTRACT_LIMIT → Tier 3 skip. - huge_token_count = DISTILL_STRUCT_EXTRACT_LIMIT + 1 - original_context = "original repo context — must flow through unchanged" with patch("lib.distill.gather_repo_files", return_value=[]), \ - patch("lib.distill.estimate_tokens_for_files", return_value=huge_token_count), \ patch("lib.distill.completion") as mock_comp: - result = maybe_distill(original_context, "Fix a bug", "anthropic/claude-sonnet-4-5", root=str(tmp_path)) + result = maybe_distill(original_context, "Fix a bug", + "anthropic/claude-sonnet-4-5", + root=str(tmp_path)) - # distill() must NOT have been called at all mock_comp.assert_not_called() - # The original context must be returned unchanged assert result[0] == original_context assert result[1] == 0 # zero input tokens used assert result[2] == 0 # zero output tokens assert result[3] == 0.0 # zero cost + + def test_overflow_in_tier2_falls_back_to_original(self, tmp_path): + """If a Tier-2 LLM call overflows the model context (or otherwise + errors), the outer try/except returns the original context. This is + the safety net we rely on for very large codebases now that the + upper cap is gone.""" + files = {"main.py": "x"} + make_git_repo(tmp_path, files) + + huge_files = [ + {"path": f"file_{i}.py", "content": "def f(): pass\n", + "is_source": True, "truncated": False} + for i in range(10) + ] + + original_context = "original — should flow through on overflow" + + with patch("lib.distill.gather_repo_files", return_value=huge_files), \ + patch("lib.distill.estimate_tokens_for_files", + return_value=DISTILL_SMALL_REPO_LIMIT * 100), \ + patch("lib.distill.completion", + side_effect=Exception("context length exceeded")): + result = maybe_distill(original_context, "Fix a bug", + "anthropic/claude-sonnet-4-5", + root=str(tmp_path)) + + assert result[0] == original_context + assert result[1] == 0 + assert result[2] == 0 + assert result[3] == 0.0 From 8e40e55dff5e4ac5bd377eca002beac30baf102e Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Wed, 20 May 2026 00:21:58 +0000 Subject: [PATCH 03/39] Fix module imports that break under production sys.path layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Workshop crashed on every run since commit e4209e7 (formatting.py extract) with: File ".remote-dev-bot/lib/workshop.py", line 31, in from lib.formatting import _fmt_tok, ... ModuleNotFoundError: No module named 'lib' The workflow heredocs invoke library modules with `sys.path.insert(0, '.remote-dev-bot/lib')` — only the lib/ directory on sys.path, not the rdb repo root. Three modules used `from lib.X import ...` which requires the rdb root to be on sys.path so that `lib` resolves as a package by name: - lib/workshop.py (line 31, broken since e4209e7) - lib/cumulative_cost.py (line 19, silently broken — wrapped in try/except so cumulative cost tables silently went missing) - lib/design_loop.py (line 13, broken since lib/tools.py extract but never noticed because every callsite also crashed earlier on the workshop import) Changed all three to sibling-style `from X import ...` to match the heredoc style. Script entry points (resolve.py, reconcile.py, post_fallback_cost.py) are unaffected — they each take care of their own sys.path with a `sys.path.insert(0, os.path.dirname( os.path.dirname(__file__)))` so they work under either layout. Tests passed because pytest runs with `PYTHONPATH=.` which puts both the rdb root AND lib/ on the path, masking the bug. Added tests/test_production_imports.py which spawns a subprocess with only lib/ on sys.path (matching production heredoc), and asserts each library module imports cleanly. Locked in for all seven non-entry- point modules. Verified against the failing run: gnovak/bridge-analysis issue #393, workshop run 26121417319 — traceback line-for-line matches. --- lib/cumulative_cost.py | 2 +- lib/design_loop.py | 2 +- lib/workshop.py | 2 +- tests/test_production_imports.py | 87 ++++++++++++++++++++++++++++++++ 4 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 tests/test_production_imports.py diff --git a/lib/cumulative_cost.py b/lib/cumulative_cost.py index 039677a..3bafdc5 100644 --- a/lib/cumulative_cost.py +++ b/lib/cumulative_cost.py @@ -16,7 +16,7 @@ import re import subprocess -from lib.formatting import _fmt_tok, _fmt_loc +from formatting import _fmt_tok, _fmt_loc def extract_costs_from_text(text: str) -> tuple: diff --git a/lib/design_loop.py b/lib/design_loop.py index 4ccb9a3..b0d039f 100644 --- a/lib/design_loop.py +++ b/lib/design_loop.py @@ -10,7 +10,7 @@ import subprocess import sys -from lib.tools import ( +from tools import ( validate_path as _tools_validate_path, execute_read_file as _tools_execute_read_file, execute_gh as _tools_execute_gh, diff --git a/lib/workshop.py b/lib/workshop.py index e107ec4..1d55a4b 100644 --- a/lib/workshop.py +++ b/lib/workshop.py @@ -28,7 +28,7 @@ # Cost formatting helpers — imported from shared lib/formatting.py # --------------------------------------------------------------------------- -from lib.formatting import _fmt_tok, _fmt_ela, _fmt_bpd, _fmt_loc, _fmt_info, TABLE_HEADER +from formatting import _fmt_tok, _fmt_ela, _fmt_bpd, _fmt_loc, _fmt_info, TABLE_HEADER def _build_cost_table(input_tokens, output_tokens, cost, elapsed, output_text): diff --git a/tests/test_production_imports.py b/tests/test_production_imports.py new file mode 100644 index 0000000..ec33670 --- /dev/null +++ b/tests/test_production_imports.py @@ -0,0 +1,87 @@ +"""Regression tests for module-level imports under the production sys.path layout. + +The remote-dev-bot.yml workflow invokes the lib/ modules with +`sys.path.insert(0, '.remote-dev-bot/lib')` — i.e., only the lib/ +directory is on sys.path, NOT the rdb repo root. Modules that import +their siblings as `from lib.X import ...` will silently break in +production while continuing to pass test suites that run with +`PYTHONPATH=.` (which adds both to the path). + +This file spawns a subprocess with sys.path set up the same way as the +workflow heredoc, then imports each module. The subprocess isolation +matters: if the test process already has both paths on sys.path, +`from lib.X` will resolve from the rdb root and mask the bug. +""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).parent.parent +LIB_DIR = REPO_ROOT / "lib" + + +def _import_in_production_pathlayout(module_name: str) -> subprocess.CompletedProcess: + """Run `python -c 'import '` with only lib/ on sys.path. + + Mimics the production invocation: workflow heredocs do + `sys.path.insert(0, '.remote-dev-bot/lib')` and then `from import ...`. + Returns the CompletedProcess so tests can assert on returncode and stderr. + """ + # Match the workflow heredoc: prepend lib/ to sys.path, but leave stdlib + # paths in place. Critically, the REPO ROOT (which would let `from lib.X` + # resolve) must NOT be on sys.path — that's the production condition we're + # reproducing. + code = ( + "import sys; " + f"sys.path = [p for p in sys.path if p not in ({str(REPO_ROOT)!r}, '')]; " + f"sys.path.insert(0, {str(LIB_DIR)!r}); " + f"import {module_name}" + ) + # Use the same Python that's running pytest. PYTHONPATH must be cleared so + # the parent process's path doesn't leak in and mask the bug. + env = {k: v for k, v in os.environ.items() if k != "PYTHONPATH"} + return subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + +class TestProductionSysPathImports: + """Each library module must import cleanly when only lib/ is on sys.path.""" + + @pytest.mark.parametrize("module_name", [ + "workshop", + "cumulative_cost", + "formatting", + "context", + "tools", + "distill", + "design_loop", + ]) + def test_module_imports_with_only_lib_on_path(self, module_name): + """The workflow heredoc puts only lib/ on sys.path. Modules must + respect that — they must not assume the rdb repo root is also there + (which would let `from lib.X import ...` work). + + If this fails for a module, change its top-level imports from + `from lib.X import Y` to `from X import Y` to match the heredoc style. + Modules that are *script entry points* (resolve.py, reconcile.py, + post_fallback_cost.py) handle this themselves with a + `sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))` — + those are not tested here because they would have to be invoked as + scripts, not imported. + """ + result = _import_in_production_pathlayout(module_name) + assert result.returncode == 0, ( + f"Importing {module_name!r} with production sys.path failed:\n" + f"--- stdout ---\n{result.stdout}\n" + f"--- stderr ---\n{result.stderr}" + ) From b02ee8478c0ef95d7694d8ce20afb09b7b126514 Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Wed, 20 May 2026 02:29:49 +0000 Subject: [PATCH 04/39] Tighten e2e assertions so missing-output cases fail instead of pass MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workshop crash fixed in PR #623 was present during the v0.9.0 e2e run, but the workshop e2e assertion was lax: both "Workshop Stage 1/2 comment found" and "no workshop comment found" counted as PASS. The crash produced an "exited unexpectedly" fallback comment (which doesn't match the Stage 1/2 substrings) → 0 matches → PASS by the catch-all. That's the kind of false-pass we need e2e to never produce. Same lax pattern was in seven other places. Tightened all of them: design (line 818-825) PASS no "Design analysis" comment found → FAIL workshop (line 826-841) PASS no Stage 1/2 comment found → FAIL build (line 855-867) PASS PR created, no council review comment → FAIL (Council review IS the value-add of build mode over plain resolve; missing it = the test didn't exercise what build is FOR.) delegate (line 868-911) PASS PR created, pipeline complete but no council → FAIL PASS PR created, no pipeline-complete marker → FAIL (Both the council review AND the pipeline-complete marker are required signals that the full pipeline ran end-to-end.) rf-review (line 1041-1052) PASS no "Code review by" comment found → FAIL timeout (line 1063-1088) PASS run completed, no watchdog comment found → FAIL (The whole point of the timeout test is to verify the watchdog fires and posts a "could not fully resolve" / "exited unexpectedly" comment. No comment = the timeout path wasn't exercised.) wrapup (line 1146-1198) PASS run completed, no clear output found → FAIL (Plus: branch search pattern was `openhands-fix-issue-$N`, stale since the v0.6.0 OpenHands removal. Current branches are `rdb-fix-issue-$N-$alias`, so the "agent finished — PR created" branch never fired and the test had been silently coasting on the "no clear output" catch-all PASS for months.) reconcile (line 977-998) PASS force-pushed but no "Reconcile complete" comment → FAIL (Force-push without the completion comment means the workflow crashed between the push and the comment — not a real success.) All substring patterns verified against the actual strings the code emits (lib/workshop.py, lib/reconcile.py, .github/workflows/ remote-dev-bot.yml). No false-negative risk from grep-pattern drift. Pytest still 655 passing. --- tests/e2e.sh | 86 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/tests/e2e.sh b/tests/e2e.sh index cd31efb..df5e7d2 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -811,29 +811,38 @@ for pos in "${!issue_nums[@]}"; do if [[ "$conclusion" == "success" ]]; then if [[ "$test_type" == "design" ]]; then - # Design mode: check if a comment was posted (not a PR) + # Design mode produces nothing but a "Design analysis" comment; + # missing that comment = the agent did not deliver its only output. comment_count=$(gh api "repos/$TEST_REPO/issues/$issue_num/comments" \ --jq '[.[] | select(.body | contains("Design analysis"))] | length' \ 2>/dev/null || echo "0") if [[ "$comment_count" -gt 0 ]]; then status="PASS (comment posted)" + ((pass++)) || true else - status="PASS (no comment found)" + status="FAIL (no design comment posted — workflow ran but did not deliver)" + ((fail++)) || true fi - ((pass++)) || true elif [[ "$test_type" == "workshop" ]]; then - # Workshop mode: check if a Stage 2 summary comment was posted (no PR) + # Workshop produces nothing but Stage 1 / Stage 2 comments; missing + # them means the workshop pipeline did not deliver. (This is exactly + # the failure mode that PR #623 fixed — workshop crashed on import, + # produced only an "exited unexpectedly" fallback, and the prior + # lax assertion still counted it as a PASS.) comment_count=$(gh api "repos/$TEST_REPO/issues/$issue_num/comments" \ --jq '[.[] | select(.body | contains("Workshop Stage 2 complete") or contains("Workshop Stage 1"))] | length' \ 2>/dev/null || echo "0") if [[ "$comment_count" -gt 0 ]]; then status="PASS (workshop comments posted)" + ((pass++)) || true else - status="PASS (no workshop comment found)" + status="FAIL (no Workshop Stage 1/2 comment — workshop did not run to completion)" + ((fail++)) || true fi - ((pass++)) || true elif [[ "$test_type" == "build" ]]; then - # Build mode: Stage 1 creates a PR; Stage 2 posts council code reviews on the PR. + # Build mode: Stage 1 creates a PR; Stage 2 posts council code reviews. + # The council review IS the value-add of build mode over plain resolve, + # so missing it = the test didn't exercise what makes build distinct. pr_num=$(gh pr list --repo "$TEST_REPO" \ --search "head:rdb-fix-issue-$issue_num" \ --json number --jq '.[0].number // empty' 2>/dev/null || echo "") @@ -849,15 +858,18 @@ for pos in "${!issue_nums[@]}"; do 2>/dev/null || echo "0") if [[ "$council_comment_count" -gt 0 ]]; then status="PASS (PR #$pr_num + council review posted)" + ((pass++)) || true else - status="PASS (PR #$pr_num created, no council comment found)" + status="FAIL (PR #$pr_num created but no council review — Stage 2 did not run)" + ((fail++)) || true fi - ((pass++)) || true fi elif [[ "$test_type" == "delegate" ]]; then # Delegate mode: Stages 1-3 post design/council/revision comments on issue; # Stage 4 creates a PR; Stage 5 posts council code review on the PR; - # final summary comment is posted on the issue. + # final "Delegate pipeline complete" comment on the issue marks completion. + # The delegate test must exercise BOTH the council review AND the final + # marker — missing either means the pipeline didn't fully run. pr_num=$(gh pr list --repo "$TEST_REPO" \ --search "head:rdb-fix-issue-$issue_num" \ --json number --jq '.[0].number // empty' 2>/dev/null || echo "") @@ -885,12 +897,17 @@ for pos in "${!issue_nums[@]}"; do 2>/dev/null || echo "0") if [[ "$pipeline_comment" -gt 0 ]] && [[ "$council_comment_count" -gt 0 ]]; then status="PASS (PR #$pr_num + council review + pipeline complete)" + ((pass++)) || true elif [[ "$pipeline_comment" -gt 0 ]]; then - status="PASS (PR #$pr_num created, pipeline complete, no council comment found)" + status="FAIL (PR #$pr_num + pipeline complete but no council review — Stage 5 missing)" + ((fail++)) || true + elif [[ "$council_comment_count" -gt 0 ]]; then + status="FAIL (PR #$pr_num + council review but no pipeline-complete marker — pipeline did not finish)" + ((fail++)) || true else - status="PASS (PR #$pr_num created, no pipeline complete comment found)" + status="FAIL (PR #$pr_num created but neither council review nor pipeline-complete marker)" + ((fail++)) || true fi - ((pass++)) || true fi else # Resolve mode: check if a PR was created @@ -970,10 +987,11 @@ else reconcile_comment=$(gh api "repos/$TEST_REPO/issues/$RECONCILE_PR_NUM/comments" --jq '[.[] | select(.body | test("Reconcile complete|reconcile complete"; "i"))] | length' 2>/dev/null || echo "0") if [[ "$reconcile_comment" -gt 0 ]]; then log " reconcile-test: PASS (force-pushed new SHA ${new_sha:0:7}, Reconcile complete comment found) $reconcile_url" + ((RECONCILE_PASS++)) || true else - log " reconcile-test: PASS (force-pushed new SHA ${new_sha:0:7}, no Reconcile complete comment found) $reconcile_url" + log " reconcile-test: FAIL (force-pushed but no 'Reconcile complete' comment — workflow may have crashed after the push) $reconcile_url" + ((RECONCILE_FAIL++)) || true fi - ((RECONCILE_PASS++)) || true fi fi @@ -1025,10 +1043,11 @@ elif [[ "$RF_REVIEW_RESULT" == "success" ]]; then 2>/dev/null || echo "0") if [[ "$comment_count" -gt 0 ]]; then rf_review_status="PASS (review comment posted)" + ((RF_PASS++)) || true else - rf_review_status="PASS (no review comment found)" + rf_review_status="FAIL (review workflow succeeded but no 'Code review by' comment posted)" + ((RF_FAIL++)) || true fi - ((RF_PASS++)) || true else rf_review_status="FAIL ($RF_REVIEW_RESULT)" ((RF_FAIL++)) || true @@ -1049,9 +1068,12 @@ if [[ -z "$TIMEOUT_RESULT" ]]; then timeout_status="TIMEOUT (e2e wait exceeded)" ((TIMEOUT_PHASE_FAIL++)) || true elif [[ "$TIMEOUT_RESULT" == "success" || "$TIMEOUT_RESULT" == "failure" ]]; then - # Both success and failure are valid: the watchdog kills OpenHands (making the - # resolver step exit non-zero → job conclusion = failure), but cleanup steps - # still run due to if:always(). Accept either conclusion as "run completed". + # Both success and failure are valid: the watchdog kills the agent (making + # the resolver step exit non-zero → job conclusion = failure), but cleanup + # steps still run due to if:always(). The PASS criterion is that the + # watchdog actually fired and posted a "could not fully resolve" / + # "exited unexpectedly" comment — without that, we have no evidence the + # timeout path was exercised at all. comment_count=$(gh api "repos/$TEST_REPO/issues/$timeout_issue_num/comments" \ --jq '[.[] | select(.body | contains("could not fully resolve") or contains("exited unexpectedly"))] | length' \ 2>/dev/null || echo "0") @@ -1059,8 +1081,8 @@ elif [[ "$TIMEOUT_RESULT" == "success" || "$TIMEOUT_RESULT" == "failure" ]]; the timeout_status="PASS (watchdog fired + comment posted) [${TIMEOUT_RESULT}]" ((TIMEOUT_PHASE_PASS++)) || true else - timeout_status="PASS (run completed, no comment found) [${TIMEOUT_RESULT}]" - ((TIMEOUT_PHASE_PASS++)) || true + timeout_status="FAIL (run completed but no watchdog comment — timeout path did not fire) [${TIMEOUT_RESULT}]" + ((TIMEOUT_PHASE_FAIL++)) || true fi else timeout_status="FAIL ($TIMEOUT_RESULT)" @@ -1144,23 +1166,33 @@ if [[ -z "$WRAPUP_RESULT" ]]; then wrapup_status="TIMEOUT (e2e wait exceeded)" ((WRAPUP_PHASE_FAIL++)) || true elif [[ "$WRAPUP_RESULT" == "success" ]]; then - # Check for either a PR (agent finished) or a failure comment (wrapup triggered) + # The wrapup test runs with max_iterations=4 on a task the agent cannot + # plausibly finish, so we expect *either*: + # - A PR (agent finished anyway), OR + # - A "could not fully resolve" comment (wrapup nudge triggered the + # graceful failure path). + # Neither = the run completed without exercising the wrapup path → FAIL. + # + # NB: branch name was historically `openhands-fix-issue-$N` (stale since + # v0.6.0 OpenHands removal); current branches are `rdb-fix-issue-$N-$alias`. + # That stale name meant the pr_count branch never fired and the test was + # silently coasting on the "no clear output" PASS for months. pr_count=$(gh pr list --repo "$TEST_REPO" \ - --search "head:openhands-fix-issue-$wrapup_issue_num" \ + --search "head:rdb-fix-issue-$wrapup_issue_num" \ --json number --jq 'length' 2>/dev/null || echo "0") comment_count=$(gh api "repos/$TEST_REPO/issues/$wrapup_issue_num/comments" \ --jq '[.[] | select(.body | contains("could not fully resolve"))] | length' \ 2>/dev/null || echo "0") if [[ "$pr_count" -gt 0 ]]; then wrapup_status="PASS (agent finished — PR created)" - cleanup_branches+=("openhands-fix-issue-$wrapup_issue_num") + cleanup_branches+=("rdb-fix-issue-$wrapup_issue_num") ((WRAPUP_PHASE_PASS++)) || true elif [[ "$comment_count" -gt 0 ]]; then wrapup_status="PASS (wrapup triggered — failure comment posted)" ((WRAPUP_PHASE_PASS++)) || true else - wrapup_status="PASS (run completed, no clear output found)" - ((WRAPUP_PHASE_PASS++)) || true + wrapup_status="FAIL (run completed but neither PR nor wrapup comment — wrapup path did not fire)" + ((WRAPUP_PHASE_FAIL++)) || true fi else wrapup_status="FAIL ($WRAPUP_RESULT)" From 93bc290291ae8a0469a6c298e1a61ea570288fee Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Sat, 23 May 2026 05:59:41 +0000 Subject: [PATCH 05/39] Make timeout e2e test reliably exercise the watchdog The most recent full-suite run failed the (now-tightened) timeout assertion because the agent scope-reduced the vague refactor task and finished in 50s instead of being killed by the 1-minute watchdog: Tool: finish({'success': True, 'explanation': 'Created utils.py with type hints and docstrings, and test_utils.py with 14 passing unit tests.'}) The test had been silently coasting on the lax "no comment found = PASS" assertion. PR #624 tightened it to FAIL, which surfaced this: the watchdog isn't actually being exercised because the task is too soft and the timeout too long. ## Fix Two changes, both needed: - **Concrete uncompressible task**: replace "refactor every file to follow best practices..." with "create exactly 50 Python files named day_01.py..day_50.py with this exact body." The agent can no longer satisfy the requirement by writing one demonstrative file; it's all 50 or explicit failure. - **Shorter watchdog**: cut timeout_minutes from 1 to 0.5. Even scope-reduced work doesn't fit in 30s. Required adding fractional support: `timeout_minutes` is now `float` in lib/config.py's ALLOWED_ARGS, and the bash arithmetic in the workflow's 5 watchdog steps changes from `$((TIMEOUT_MINUTES*60))` (bash integer math, drops fractional values to 0) to `awk -v m="$TIMEOUT_MINUTES" 'BEGIN{print int(m*60)}'` (handles int and float, rounds down to whole seconds). The float type also leaves the door open for users who want timeout_minutes=1.5 or similar in production configs. 655 unit tests still pass. --- .github/workflows/remote-dev-bot.yml | 10 +++++----- lib/config.py | 2 +- tests/e2e.sh | 19 ++++++++++++++++--- 3 files changed, 22 insertions(+), 9 deletions(-) diff --git a/.github/workflows/remote-dev-bot.yml b/.github/workflows/remote-dev-bot.yml index a191e69..81264ff 100644 --- a/.github/workflows/remote-dev-bot.yml +++ b/.github/workflows/remote-dev-bot.yml @@ -311,7 +311,7 @@ jobs: fi TIMEOUT_MINUTES="${{ needs.parse.outputs.timeout_minutes }}" - TIMEOUT_SECONDS=$((TIMEOUT_MINUTES * 60)) + TIMEOUT_SECONDS=$(awk -v m="$TIMEOUT_MINUTES" 'BEGIN{print int(m*60)}') echo "Resolver timeout: ${TIMEOUT_MINUTES} minutes" echo $(date +%s) > /tmp/start_time # Run agent loop under timeout. On expiry: SIGTERM first, then SIGKILL after 60s. @@ -863,7 +863,7 @@ jobs: fi TIMEOUT_MINUTES="${{ needs.parse.outputs.timeout_minutes }}" - TIMEOUT_SECONDS=$((TIMEOUT_MINUTES * 60)) + TIMEOUT_SECONDS=$(awk -v m="$TIMEOUT_MINUTES" 'BEGIN{print int(m*60)}') echo "Resolver timeout: ${TIMEOUT_MINUTES} minutes" echo $(date +%s) > /tmp/start_time # Run agent loop under timeout. On expiry: SIGTERM first, then SIGKILL after 60s. @@ -2204,7 +2204,7 @@ jobs: echo "Reconciling PR #${PR_NUMBER} onto base branch: ${BASE_BRANCH}" TIMEOUT_MINUTES="${{ needs.parse.outputs.timeout_minutes }}" - TIMEOUT_SECONDS=$((TIMEOUT_MINUTES * 60)) + TIMEOUT_SECONDS=$(awk -v m="$TIMEOUT_MINUTES" 'BEGIN{print int(m*60)}') echo "Reconcile timeout: ${TIMEOUT_MINUTES} minutes" echo $(date +%s) > /tmp/start_time @@ -3742,7 +3742,7 @@ jobs: Running resolve agent with \`${{ needs.parse.outputs.alias }}\` (\`${{ needs.parse.outputs.model }}\`) to implement the revised design..." TIMEOUT_MINUTES="${{ needs.parse.outputs.timeout_minutes }}" - TIMEOUT_SECONDS=$((TIMEOUT_MINUTES * 60)) + TIMEOUT_SECONDS=$(awk -v m="$TIMEOUT_MINUTES" 'BEGIN{print int(m*60)}') echo "Resolver timeout: ${TIMEOUT_MINUTES} minutes" timeout --kill-after=60 "$TIMEOUT_SECONDS" \ @@ -4064,7 +4064,7 @@ jobs: Running resolve agent with \`${{ needs.parse.outputs.alias }}\` (\`${{ needs.parse.outputs.model }}\`) to apply council code review feedback to this PR..." TIMEOUT_MINUTES="${{ needs.parse.outputs.timeout_minutes }}" - TIMEOUT_SECONDS=$((TIMEOUT_MINUTES * 60)) + TIMEOUT_SECONDS=$(awk -v m="$TIMEOUT_MINUTES" 'BEGIN{print int(m*60)}') echo "Stage 6 resolver timeout: ${TIMEOUT_MINUTES} minutes" timeout --kill-after=60 "$TIMEOUT_SECONDS" \ diff --git a/lib/config.py b/lib/config.py index 9287e15..2fe866a 100644 --- a/lib/config.py +++ b/lib/config.py @@ -89,7 +89,7 @@ def normalize_config(config): ALLOWED_ARGS = { "max_iterations": int, # agent.max_iterations (code-writing stages) "max_design_iterations": int, # delegate: budget for design/exploration stages - "timeout_minutes": int, # agent.timeout_minutes + "timeout_minutes": float, # agent.timeout_minutes (float so e2e tests can use sub-minute values) "extra_files": list, # mode's extra_files "branch": str, # agent.branch (target branch for PRs) "status_log_interval": int, # rolling status log interval (0 = disabled) diff --git a/tests/e2e.sh b/tests/e2e.sh index df5e7d2..4331471 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -375,17 +375,30 @@ log " Issue #$RF_ISSUE_NUM created. Triggering /agent-resolve..." post_issue_comment "$RF_ISSUE_NUM" "$TEST_REPO" "/agent-resolve" # --- Trigger timeout test --- +# Purpose: verify the watchdog kills the agent process when it exceeds +# timeout_minutes. The task must be concrete enough that the agent cannot +# scope-reduce its way to an early finish() — and the timeout short enough +# that even fast scope-reduced work doesn't fit. (The previous vague +# "refactor everything" prompt with timeout_minutes=1 let the agent finish +# in 50s by creating one file with type hints; watchdog never fired.) log "Creating timeout test issue..." timeout_ts=$(date +%s) timeout_title="Test: timeout enforcement (e2e-timeout-$timeout_ts)" timeout_issue_url=$(gh issue create --repo "$TEST_REPO" \ --title "$timeout_title" \ - --body "Analyze and refactor every file in this repository to follow best practices, add comprehensive type hints, docstrings, and unit tests. This task is intentionally scope-heavy.") + --body "Create exactly 50 Python files named day_01.py through day_50.py in a new \`days/\` directory at the repo root. Each file must contain exactly one function with this exact signature and body (substitute NN with the day number, NOT zero-padded in the body): + +\`\`\`python +def visits_on_day_NN() -> int: + return NN +\`\`\` + +All 50 files must be created and committed in a single PR. This is a concrete, fully-specified task that cannot be scope-reduced — produce all 50 files or report failure.") timeout_issue_num="${timeout_issue_url##*/}" cleanup_issues+=("$timeout_issue_num") -log " Issue #$timeout_issue_num. Posting /agent-resolve with timeout_minutes = 1..." -post_issue_comment "$timeout_issue_num" "$TEST_REPO" $'/agent-resolve\ntimeout_minutes = 1' +log " Issue #$timeout_issue_num. Posting /agent-resolve with timeout_minutes = 0.5..." +post_issue_comment "$timeout_issue_num" "$TEST_REPO" $'/agent-resolve\ntimeout_minutes = 0.5' TIMEOUT_RUN_ID="" TIMEOUT_RESULT="" From 43aae99f46ab596ad994d4b271732b3a99d961df Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Sat, 23 May 2026 06:05:59 +0000 Subject: [PATCH 06/39] Distillation: tighten prompt, fix metric, model-dependent Tier-1 limit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three independent fixes investigated for bridge-analysis #392, where distillation compressed leaderboard.py down to 972 tokens (entire file → ~snippets) and the status log misleadingly reported "5.6K → 972 tokens, ~$0.05 saved". ## 1. Tighten the distill prompt against over-excerpting When an issue references specific line numbers (e.g., "lines 312-313") the distill LLM was extracting tiny snippets and discarding the surrounding file. Prompt now has a hard rule: files to be modified are included in FULL, regardless of how specific the task is. Also addressed the specific line-number-excerpting failure mode in the prompt text. The user-message instructions and system prompt are now consistent on this (no more "[full content or relevant excerpt]" escape hatch). ## 2. Fix the metric (pre_tokens reflects what distillation saw) The status log read: "Distillation: 5.6K → 972 tokens (4.6K saved/iter × 30 iters)" That 5.6K was the size of EXTRA_FILES (README, AGENTS.md, etc.) — NOT what distillation processed. The codebase distillation actually scanned was 368K tokens. maybe_distill() now returns the codebase total as a 6th tuple element so resolve.py can pass the correct value into build_distillation_summary(). Bridge-analysis #392 same numbers, before/after this fix: OLD: 5.6K → 972 tokens, ~$0.05 saved NEW: 368K → 972 tokens, ~$4.30 saved ## 3. Model-dependent Tier-1-vs-Tier-2 cutoff DISTILL_SMALL_REPO_LIMIT was a hardcoded 100K, set when 200K-window models were typical. Modern models we configure (claude-sonnet-4-6, claude-opus-4-7, gpt-5.5, gemini-2.5-flash) all have 1M-token input windows, so we can comfortably send much larger codebases to a single Tier-1 LLM call. Added _small_repo_limit(model) which scales the cutoff to 25% of the model's input window, clamped to [50K, 500K]. Old 100K constant is the fallback for models LiteLLM doesn't know. Effective Tier-1 ceiling per model: - claude-sonnet-4-6, opus-4-7: 250K (1M × 25%) - gpt-5.5: 262K (1.05M × 25%) - gemini-2.5-flash: 262K - claude-sonnet-4-5 (200K win): 50K (floored) - unknown / not in LiteLLM: 100K (legacy default) 662 tests pass (was 655; +7 for _small_repo_limit + prompt content). --- lib/design_loop.py | 5 +- lib/distill.py | 118 +++++++++++++++++++++++++++++++----------- lib/resolve.py | 19 ++++--- tests/test_distill.py | 89 ++++++++++++++++++++++++++++--- 4 files changed, 186 insertions(+), 45 deletions(-) diff --git a/lib/design_loop.py b/lib/design_loop.py index b0d039f..44b2a90 100644 --- a/lib/design_loop.py +++ b/lib/design_loop.py @@ -338,7 +338,10 @@ def run_design_loop( + (f"\n\n## Discussion so far:\n{issue_comments}" if issue_comments else "") ) print("Running context distillation pre-step (design loop)...") - distilled, distill_input_tokens, distill_output_tokens, distill_cost, structural_extract = maybe_distill( + # Note: design loop currently doesn't surface a per-iter savings + # metric, so we discard codebase_total_tokens here. + (distilled, distill_input_tokens, distill_output_tokens, + distill_cost, structural_extract, _codebase_total) = maybe_distill( extra_context, issue_context_text, model ) if distilled != extra_context: diff --git a/lib/distill.py b/lib/distill.py index e343022..aa9fd15 100644 --- a/lib/distill.py +++ b/lib/distill.py @@ -28,9 +28,18 @@ # --- Constants (all hardcoded — no user-facing config) --- -# Token budget below which we skip the structural-extract step and just send -# the whole codebase to the distill LLM in one shot. -DISTILL_SMALL_REPO_LIMIT = 100_000 # tokens — try to send whole codebase +# Default token budget below which we skip the structural-extract step and +# send the whole codebase to the distill LLM in one shot. Used as a fallback +# when the model's context window can't be looked up. For models with a known +# context window, the effective limit is computed by _small_repo_limit() as +# DISTILL_SMALL_REPO_BUDGET_FRACTION × context_window — clamped between +# DISTILL_SMALL_REPO_LIMIT_MIN and DISTILL_SMALL_REPO_LIMIT_MAX. This way a +# 1M-token model gets the headroom to ingest a much bigger codebase in one +# call, while a smaller model stays conservative. +DISTILL_SMALL_REPO_LIMIT = 100_000 # fallback for unknown models +DISTILL_SMALL_REPO_LIMIT_MIN = 50_000 # floor (don't go below this) +DISTILL_SMALL_REPO_LIMIT_MAX = 500_000 # ceiling (don't get carried away) +DISTILL_SMALL_REPO_BUDGET_FRACTION = 0.25 # share of input window we'll spend on codebase # Per-file size caps before truncation (in characters) SOURCE_FILE_CAP = 50_000 # for source/config files @@ -85,6 +94,31 @@ STRUCT_EXTRACT_HEAD_LINES = 30 +# --- Model-dependent thresholds --- + +def _small_repo_limit(model): + """Return the Tier-1-vs-Tier-2 cutoff for this model. + + Larger context windows (1M tokens for current Claude/GPT/Gemini frontier + models) can comfortably ingest more of the codebase in one shot, so the + "small repo" threshold scales with the model's input window. Clamped to + [DISTILL_SMALL_REPO_LIMIT_MIN, DISTILL_SMALL_REPO_LIMIT_MAX]. + + Returns DISTILL_SMALL_REPO_LIMIT (the legacy 100K default) for any model + LiteLLM doesn't know about. + """ + try: + import litellm + info = litellm.get_model_info(model) + window = int(info.get("max_input_tokens") or 0) + except Exception: + window = 0 + if window <= 0: + return DISTILL_SMALL_REPO_LIMIT + raw = int(window * DISTILL_SMALL_REPO_BUDGET_FRACTION) + return max(DISTILL_SMALL_REPO_LIMIT_MIN, min(DISTILL_SMALL_REPO_LIMIT_MAX, raw)) + + # --- File gathering --- def _should_skip_path(path): @@ -298,24 +332,41 @@ def format_structural_extract(files, include_line_numbers=False): DISTILL_SYSTEM_PROMPT = """\ You are a code relevance filter. Given a codebase and a task description, \ -extract ONLY what is directly relevant to implementing the task: -- Full content of files that will need to be modified -- Signatures and docstrings (not full bodies) of functions that provide context \ -but won't be directly modified -- Schema definitions, data structures, or config formats the task touches -- Brief explanation of how each included item relates to the task +produce a focused context block for a downstream coding agent. + +## Hard rule: files to be modified must be included in FULL + +For any file the agent will need to modify, include the COMPLETE file content \ +verbatim — every line, top to bottom. Do NOT excerpt, summarize, or skip lines \ +even if the task references specific line numbers or function names. The agent \ +needs the surrounding code (imports, helpers, module-level state, neighbouring \ +functions) to understand how its changes interact with the rest of the file. + +A request like "drop the threshold check in `_player_stats` at line 312" still \ +requires the FULL file — the agent has to see what `_player_stats` calls, what \ +calls `_player_stats`, what state it depends on, and so on. Returning only the \ +lines around 312 is worse than returning nothing: it gives the agent false \ +confidence that it has the relevant context when it doesn't. + +## What else to include + +- Signatures and docstrings (not full bodies) of functions in OTHER files \ +that the agent will need to call or whose interfaces it must respect. +- Schema definitions, data structures, or config formats the task touches. +- A one-line "why this file matters" note for each included file. + +## What to omit -Your goal is to reduce the need for the downstream coding agent to explore and \ -understand the whole codebase. The agent should be able to read your output and \ -know exactly which files to modify, which functions to call, and what interfaces \ -to respect — without needing to read anything else. +- Files unrelated to the task. +- Full bodies of helper functions in non-modified files (signatures + \ +docstrings only). -Be explicit about boundaries: "you need to read this file", "you need this \ -function signature but don't need the full body", "don't bother reading X, \ -it's not relevant to this task". +Be explicit about boundaries: \"you need to read this file\", \"you need \ +this function signature but don't need the full body\", \"don't bother \ +reading X, it's not relevant to this task\". -Omit everything unrelated to the task. Be concise — the output feeds directly \ -into a coding agent's working context.""" +Be concise on context, but never trim files that will be modified — \ +their full content is the foundation the agent's work rests on.""" def distill(codebase_text, task_context, model): @@ -332,10 +383,12 @@ def distill(codebase_text, task_context, model): f"## Task\n\n{task_context}\n\n" f"## Codebase\n\n{codebase_text}\n\n" "## Instructions\n\n" - "Extract only what is relevant to the task above. Format as:\n\n" + "Produce a focused context block for the downstream coding " + "agent. Format as:\n\n" "### Relevant Files\n\n" "**`path/to/file.py`** — [one sentence: why this file matters]\n" - "```python\n[full content or relevant excerpt]\n```\n\n" + "```python\n[FULL file content, verbatim, no excerpting — see " + "the system prompt's hard rule]\n```\n\n" "### Key Interfaces\n\n" "[signatures and docstrings from other files that the task touches, " "without full bodies]\n\n" @@ -424,13 +477,16 @@ def _identify_relevant_files(extract_text, task_context, model): def maybe_distill(repo_context, issue_context, model, root="."): """Run context distillation pre-step. - Returns (context_text, input_tokens, output_tokens, cost, structural_extract). - On failure or skip, returns (repo_context, 0, 0, 0.0, ""). - The structural_extract is a compact index of all functions/classes with line - numbers, intended to be included in the agent's system prompt alongside the - distilled context. + Returns (context_text, input_tokens, output_tokens, cost, + structural_extract, codebase_total_tokens). + On failure or skip, returns (repo_context, 0, 0, 0.0, "", 0). + The structural_extract is a compact index of all functions/classes with + line numbers, intended to be included in the agent's system prompt + alongside the distilled context. codebase_total_tokens is the estimated + size of the codebase that distillation scanned, so callers can compute + meaningful "tokens-saved-per-iteration" metrics. """ - fallback = (repo_context, 0, 0, 0.0, "") + fallback = (repo_context, 0, 0, 0.0, "", 0) try: files = gather_repo_files(root=root) @@ -454,9 +510,11 @@ def maybe_distill(repo_context, issue_context, model, root="."): total_output = 0 total_cost = 0.0 - if total_tokens <= DISTILL_SMALL_REPO_LIMIT: - # Tier 1: Small repo — send full codebase - print(" [Distill] Tier 1 (small repo): sending full codebase") + small_repo_limit = _small_repo_limit(model) + if total_tokens <= small_repo_limit: + # Tier 1: Small repo (relative to model's context window) — send + # full codebase in one call. + print(f" [Distill] Tier 1 (small repo, ≤{small_repo_limit:,} tokens for this model): sending full codebase") codebase_text = format_codebase(files) result, inp, out, cost = distill(codebase_text, issue_context, model) total_input += inp @@ -507,7 +565,7 @@ def maybe_distill(repo_context, issue_context, model, root="."): return fallback print(f" [Distill] Distillation complete: ~{len(result) // 4:,} tokens output") - return result, total_input, total_output, total_cost, agent_structural_extract + return result, total_input, total_output, total_cost, agent_structural_extract, total_tokens except Exception as e: print(f" [Distill] Distillation failed: {e} — proceeding with full repo context") diff --git a/lib/resolve.py b/lib/resolve.py index 8e9a71f..ca32d1c 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -1145,16 +1145,20 @@ def main(): distill_input_tokens = 0 distill_output_tokens = 0 distill_cost = 0.0 - pre_distill_tokens = 0 # token estimate before distillation - post_distill_tokens = 0 # token estimate after distillation (0 = distillation did not run) + # pre_distill_tokens reflects what distillation actually saw (the codebase + # total), so the savings metric in the status log compares against the + # hypothetical "drag full codebase along every iteration" baseline. It is + # NOT the size of EXTRA_FILES (which was a bug — those are added context + # docs, not the thing distillation is replacing). + pre_distill_tokens = 0 + post_distill_tokens = 0 # 0 = distillation did not run if DISTILL_ENABLED: try: from lib.distill import maybe_distill - pre_distill_tokens = len(repo_context) // 4 print("Running context distillation pre-step...") - # Estimate tokens before distillation so we can measure savings - undistilled_tokens = estimate_tokens([{"content": repo_context}]) - distilled, distill_input_tokens, distill_output_tokens, distill_cost, structural_extract = maybe_distill( + (distilled, distill_input_tokens, distill_output_tokens, + distill_cost, structural_extract, + codebase_total_tokens) = maybe_distill( repo_context, issue_context, LLM_MODEL ) if distilled != repo_context: @@ -1165,9 +1169,10 @@ def main(): if structural_extract: agent_context += f"\n\n## Codebase Index\n\n{structural_extract}" distillation_ran = True + pre_distill_tokens = codebase_total_tokens post_distill_tokens = len(distilled) // 4 print(f"Distillation complete: {distill_input_tokens} input tokens, {distill_output_tokens} output tokens, ${distill_cost:.4f}") - print(f" [Distillation] {pre_distill_tokens} tokens -> {post_distill_tokens} tokens") + print(f" [Distillation] codebase {pre_distill_tokens:,} tokens -> distilled context {post_distill_tokens:,} tokens") else: agent_context = repo_context print("Distillation skipped or returned original context") diff --git a/tests/test_distill.py b/tests/test_distill.py index f390770..709d0f9 100644 --- a/tests/test_distill.py +++ b/tests/test_distill.py @@ -462,7 +462,9 @@ def test_medium_repo_uses_structural_extract(self, tmp_path): assert result[2] == 150 # 50 + 100 def test_returns_tuple(self, tmp_path): - """maybe_distill returns (context, input_tokens, output_tokens, cost, structural_extract).""" + """maybe_distill returns + (context, input_tokens, output_tokens, cost, structural_extract, + codebase_total_tokens).""" files = {"main.py": "x"} make_git_repo(tmp_path, files) @@ -475,13 +477,16 @@ def test_returns_tuple(self, tmp_path): mock_response._hidden_params = {"response_cost": 0.001} with patch("lib.distill.completion", return_value=mock_response): - ctx, inp, out, cost, struct_extract = maybe_distill("repo", "task", "anthropic/claude-sonnet-4-5", root=str(tmp_path)) + ctx, inp, out, cost, struct_extract, codebase_tokens = maybe_distill( + "repo", "task", "anthropic/claude-sonnet-4-5", root=str(tmp_path), + ) assert ctx == "distilled" assert inp == 10 assert out == 5 assert cost == 0.001 assert "" in struct_extract + assert codebase_tokens >= 0 # --------------------------------------------------------------------------- @@ -510,6 +515,73 @@ def test_caps_are_positive(self): assert DISTILL_OUTPUT_TOKENS > 0 +# --------------------------------------------------------------------------- +# Distillation system prompt — content locks +# --------------------------------------------------------------------------- + +class TestDistillSystemPrompt: + """The distill LLM was over-compressing files when issues mentioned + specific line numbers — see PR that introduced this test for the + bridge-analysis #392 incident. The prompt now has a hard rule that + files to be modified are returned in FULL. Lock that in.""" + + def test_includes_files_must_be_full_rule(self): + from lib.distill import DISTILL_SYSTEM_PROMPT + # The phrase doesn't have to be exact — just confirm the rule + # is still present in some form that mentions "FULL" or "complete" + # for files-to-be-modified. + prompt_lower = DISTILL_SYSTEM_PROMPT.lower() + assert "full" in prompt_lower or "complete file" in prompt_lower + assert "modified" in prompt_lower or "modify" in prompt_lower + + def test_warns_against_line_number_excerpting(self): + from lib.distill import DISTILL_SYSTEM_PROMPT + # Verify the prompt addresses the specific failure mode where + # the LLM excerpts around line-number references in the task. + assert "line number" in DISTILL_SYSTEM_PROMPT.lower() + + +# --------------------------------------------------------------------------- +# _small_repo_limit (model-dependent threshold) +# --------------------------------------------------------------------------- + +class TestSmallRepoLimit: + """The Tier-1-vs-Tier-2 cutoff scales with the model's input window so + 1M-token models get the headroom to ingest a much bigger codebase in + one shot, while smaller-window models stay conservative.""" + + def test_returns_fallback_for_unknown_model(self): + from lib.distill import _small_repo_limit, DISTILL_SMALL_REPO_LIMIT + with patch("litellm.get_model_info", side_effect=Exception("unknown")): + assert _small_repo_limit("nonsense/x") == DISTILL_SMALL_REPO_LIMIT + + def test_scales_with_model_window(self): + from lib.distill import _small_repo_limit + # 1M window × 0.25 = 250K, within the clamp range + with patch("litellm.get_model_info", + return_value={"max_input_tokens": 1_000_000}): + assert _small_repo_limit("any/model") == 250_000 + + def test_clamps_to_min_for_small_windows(self): + from lib.distill import _small_repo_limit, DISTILL_SMALL_REPO_LIMIT_MIN + # 100K window × 0.25 = 25K, but we floor at MIN + with patch("litellm.get_model_info", + return_value={"max_input_tokens": 100_000}): + assert _small_repo_limit("any/model") == DISTILL_SMALL_REPO_LIMIT_MIN + + def test_clamps_to_max_for_huge_windows(self): + from lib.distill import _small_repo_limit, DISTILL_SMALL_REPO_LIMIT_MAX + # 10M window × 0.25 = 2.5M, but we ceiling at MAX + with patch("litellm.get_model_info", + return_value={"max_input_tokens": 10_000_000}): + assert _small_repo_limit("any/model") == DISTILL_SMALL_REPO_LIMIT_MAX + + def test_returns_fallback_when_window_is_zero(self): + from lib.distill import _small_repo_limit, DISTILL_SMALL_REPO_LIMIT + with patch("litellm.get_model_info", return_value={"max_input_tokens": 0}): + assert _small_repo_limit("any/model") == DISTILL_SMALL_REPO_LIMIT + + # --------------------------------------------------------------------------- # compress_linked_issue # --------------------------------------------------------------------------- @@ -741,17 +813,17 @@ class TestMaybeDistillThresholds: """ def test_above_threshold_distill_called_and_result_returned(self, tmp_path): - """Tier 1 (small repo, <= DISTILL_SMALL_REPO_LIMIT tokens): distill() is called. + """Tier 1 (small repo, <= effective small-repo limit): distill() is called. Input clearly within the small-repo limit so maybe_distill() must invoke distill() exactly once (full-codebase path) and return its output. + + The limit is now model-dependent (see _small_repo_limit); we stub it + here to keep the test independent of LiteLLM's pricing table. """ files = {"main.py": "def foo(): pass"} make_git_repo(tmp_path, files) - # Use a token estimate clearly below DISTILL_SMALL_REPO_LIMIT → Tier 1 path. - small_token_count = DISTILL_SMALL_REPO_LIMIT - 1 - mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "distilled result" @@ -762,8 +834,11 @@ def test_above_threshold_distill_called_and_result_returned(self, tmp_path): original_context = "original repo context" + # Pin both the codebase size estimate AND the model-dependent + # small-repo limit so the test deterministically exercises Tier 1. with patch("lib.distill.gather_repo_files", return_value=[{"path": "main.py", "content": "def foo(): pass", "is_source": True, "truncated": False}]), \ - patch("lib.distill.estimate_tokens_for_files", return_value=small_token_count), \ + patch("lib.distill.estimate_tokens_for_files", return_value=50_000), \ + patch("lib.distill._small_repo_limit", return_value=100_000), \ patch("lib.distill.completion", return_value=mock_response) as mock_comp: result = maybe_distill(original_context, "Fix a bug", "anthropic/claude-sonnet-4-5", root=str(tmp_path)) From 1ccd5b320864066e2b553ddecb073aef0f676a8a Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Wed, 27 May 2026 07:07:47 +0000 Subject: [PATCH 07/39] Delegate cost reporting: use canonical cumulative format, drop bespoke aggregate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the design discussion: delegate should look like calling the individual steps in sequence. Per-step cost on Stage 1, 3, 3a, 3c already worked. What was wrong was the trailing "Delegate Stages 1-3c Aggregate Cost" block, which: 1. Used a bespoke label format different from individual /agent-* invocations. 2. Contained a per-step-format cost table (with bold **$X.XX**) that the cumulative-cost scanner would have picked up as a prior step cost — guaranteeing quadratic double-counting on any later /agent-* on the same issue. Replaced with: each per-step cost block in delegate now carries a canonical cumulative table appended (via compute_cumulative_table from lib/cumulative_cost.py — the same code individual invocations use). Cumulative is empty on Stage 1 (no prior cost markers); from Stage 3 onward it includes all prior delegate stages. The cumulative on the last reported stage IS the running total, no aggregate block needed. Council reviewer comments (Stage 2, 3b) intentionally remain cost-table-less — those costs are small and the user has decided they would just be noise. Their costs won't appear in the cumulative, which matches the "what shows in the cumulative is exactly what was reported per-step" rule. No new cost-reporting code; this just routes delegate through the existing per-step + cumulative path that resolve/design/review already use. The `_build_stage_cost_block` helper added in workshop.py is a thin wrapper that calls _build_cost_table + compute_cumulative_table. run_delegate gains two parameters (github_repo, issue_number) so the cumulative scanner can fetch prior comments. The workflow passes them in from the same env vars it already uses elsewhere. 662 tests pass. --- .github/workflows/remote-dev-bot.yml | 2 + lib/workshop.py | 92 ++++++++++++++++++++++------ 2 files changed, 74 insertions(+), 20 deletions(-) diff --git a/.github/workflows/remote-dev-bot.yml b/.github/workflows/remote-dev-bot.yml index 81264ff..331ce98 100644 --- a/.github/workflows/remote-dev-bot.yml +++ b/.github/workflows/remote-dev-bot.yml @@ -3618,6 +3618,8 @@ jobs: post_comment_fn=post_comment, design_rounds=design_rounds, distill_enabled=distill_enabled, + github_repo=github_repo, + issue_number=issue_number, ) # Save usage data diff --git a/lib/workshop.py b/lib/workshop.py index 1d55a4b..8495d17 100644 --- a/lib/workshop.py +++ b/lib/workshop.py @@ -48,6 +48,53 @@ def _build_cost_table(input_tokens, output_tokens, cost, elapsed, output_text): return '\n'.join(lines) +def _build_stage_cost_block( + *, + github_repo, + issue_number, + input_tokens, + output_tokens, + cost, + elapsed, + output_text, +): + """Build the per-step cost table for a delegate stage, followed by the + canonical cumulative cost table (only when there are prior cost markers + on the issue — i.e., this is not the first step to emit a cost block). + + The cumulative table is produced by lib/cumulative_cost.compute_cumulative_table, + which is the same code paths individual /agent-resolve / /agent-design / + /agent-review invocations use, so a delegate run looks identical to the + sequence of manual invocations it expands into. + """ + parts = [_build_cost_table( + input_tokens=input_tokens, + output_tokens=output_tokens, + cost=cost, + elapsed=elapsed, + output_text=output_text, + )] + if github_repo and issue_number: + try: + from cumulative_cost import compute_cumulative_table + cum = compute_cumulative_table( + repo=github_repo, + number=str(issue_number), + current_cost=cost, + current_input_tokens=input_tokens, + current_output_tokens=output_tokens, + ) + if cum: + parts.append('') + parts.append(cum) + except Exception as e: + # Cumulative-cost computation is best-effort visibility — never + # block a stage from posting its result if it fails. + import sys as _sys + print(f" [delegate] Could not compute cumulative cost: {e}", file=_sys.stderr) + return '\n'.join(parts) + + # --------------------------------------------------------------------------- # API key resolution # --------------------------------------------------------------------------- @@ -975,6 +1022,8 @@ def run_delegate( post_comment_fn=None, design_rounds=1, distill_enabled=True, + github_repo="", + issue_number="", ): """Run the full delegate pipeline (6 stages, no human checkpoints). @@ -1123,7 +1172,9 @@ def post(body): "total_cost": all_cost, } - design_cost_table = _build_cost_table( + design_cost_block = _build_stage_cost_block( + github_repo=github_repo, + issue_number=issue_number, input_tokens=design_result.get("input_tokens", 0), output_tokens=design_result.get("output_tokens", 0), cost=design_result.get("cost", 0.0), @@ -1135,7 +1186,7 @@ def post(body): f"{design_analysis}\n\n" f"---\n" f"_Design analysis by `/agent-delegate` Stage 1 (`{model_alias}`)_\n\n" - f"{design_cost_table}" + f"{design_cost_block}" ) # ======================================================================= @@ -1267,7 +1318,9 @@ def _run_single_review(council_model): "`/agent` command(s). Blocked for safety." ) else: - revision_cost_table = _build_cost_table( + revision_cost_block = _build_stage_cost_block( + github_repo=github_repo, + issue_number=issue_number, input_tokens=revision_result["input_tokens"], output_tokens=revision_result["output_tokens"], cost=revision_result["cost"], @@ -1279,7 +1332,7 @@ def _run_single_review(council_model): f"{revised_design}\n\n" f"---\n" f"_Revised design by `/agent-delegate` Stage 3 (`{model_alias}`)_\n\n" - f"{revision_cost_table}" + f"{revision_cost_block}" ) # ======================================================================= @@ -1372,7 +1425,9 @@ def _run_single_review(council_model): "total_cost": all_cost, } - spec_cost_table = _build_cost_table( + spec_cost_block = _build_stage_cost_block( + github_repo=github_repo, + issue_number=issue_number, input_tokens=spec_result.get("input_tokens", 0), output_tokens=spec_result.get("output_tokens", 0), cost=spec_result.get("cost", 0.0), @@ -1384,7 +1439,7 @@ def _run_single_review(council_model): f"{implementation_spec}\n\n" f"---\n" f"_Implementation spec by `/agent-delegate` Stage 3a (`{model_alias}`)_\n\n" - f"{spec_cost_table}" + f"{spec_cost_block}" ) # ---- Stage 3b: Council spec review ---- @@ -1529,7 +1584,9 @@ def _run_single_spec_review(council_model): ) revised_spec = None else: - spec_revision_cost_table = _build_cost_table( + spec_revision_cost_block = _build_stage_cost_block( + github_repo=github_repo, + issue_number=issue_number, input_tokens=spec_revision_result["input_tokens"], output_tokens=spec_revision_result["output_tokens"], cost=spec_revision_result["cost"], @@ -1541,7 +1598,7 @@ def _run_single_spec_review(council_model): f"{revised_spec}\n\n" f"---\n" f"_Revised implementation spec by `/agent-delegate` Stage 3c (`{model_alias}`)_\n\n" - f"{spec_revision_cost_table}" + f"{spec_revision_cost_block}" ) # ======================================================================= @@ -1565,18 +1622,13 @@ def _run_single_spec_review(council_model): f"Proceeding to implementation (Stage 4)...\n" ) - # Build aggregate cost table for the pre-resolve stages - stages_pre_resolve_cost_table = _build_cost_table( - input_tokens=all_input_tokens, - output_tokens=all_output_tokens, - cost=all_cost, - elapsed=time.time() - design_start, - output_text=revised_spec or revised_design, - ) - post( - f"### 📊 Delegate {stages_done_label} Aggregate Cost\n\n" - f"{stages_pre_resolve_cost_table}" - ) + # No separate aggregate-cost block here — each per-stage cost block above + # carries a canonical cumulative table (via compute_cumulative_table), so + # the cumulative on the last reported stage IS the running total. The old + # bespoke "Delegate Stages 1-3c Aggregate Cost" block used the per-step + # cost format (with bold **$X.XX**), which the cumulative-cost scanner + # would have picked up as a prior per-step cost — risking quadratic + # double-counting on any subsequent /agent-* invocation on the issue. return { "design_result": design_result, From c01ca3748133fc49203899fb9f540aaf7802bb07 Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Wed, 27 May 2026 07:11:55 +0000 Subject: [PATCH 08/39] Resolve: teach the prompt to treat design/spec comments as binding scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridge-analysis #436 surfaced the failure mode: vague issue body ("we are getting to the point that I would like to launch this as a web app...") + detailed spec in /agent-delegate comments → agent scope-reduces to a tractable subset (schema + CLI), opens PR #437 (3 files, 40 LOC), declares victory. 85% of the spec dropped on the floor. The fix is NOT delegate-specific. Manual /agent-design → /agent-resolve would hit the same shortcut: the resolve system prompt biases toward small fixes ("a simple fix should take 5-10 iterations; a complex multi-file change rarely needs more than 20-30", "modify existing files rather than creating new ones", "call finish() as soon as the task is done"). For a spec-driven implementation those are actively wrong. ## What changes 1. New SCOPE section in the prompt (wired into build_system_prompt between AGENT_ROLE and the budget paragraph). Says: "If the comments contain a substantial design analysis, implementation spec, or output from a prior /agent-design, /agent-workshop, or /agent-delegate run, treat the most recent revised version as the binding contract for your work — NOT the issue body alone." Plus an explicit anti-shortcut nudge: "Scope-reducing to 'the foundational piece' and shipping a tiny PR is the wrong outcome — it leaves the issue effectively unresolved..." 2. Iteration-budget paragraph acknowledges spec-driven scope: "Implementing a detailed multi-component spec (see ## Scope above): 50-100+ iterations — every file, route, table, and test the spec lists has to be created. Stopping at 20-30 to ship a 'foundational' subset is the wrong call when a spec is the contract." 3. WORKFLOW step 3 ("Implement") softens "modify existing files rather than creating new ones" to apply only to bug fixes — for spec-driven work, the spec defines the file set. ## Why one prompt, not a delegate-specific one Delegate is a macro: it expands into the same agentic calls a user could invoke manually. The user's stated preference: delegate should look like calling the steps in sequence. If we put the scope rule in a delegate-only Stage 4 prompt, the two paths drift and the manual flow keeps shortcutting. Putting it in resolve's general prompt means it triggers for both, no separate prompt to keep in sync. 8 new tests pin the SCOPE wording + budget-paragraph update + the fact that SCOPE is actually wired into build_system_prompt's output. 670 unit tests total pass (was 662). --- lib/resolve.py | 43 +++++++++++-- tests/test_resolve_scope_prompt.py | 96 ++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 tests/test_resolve_scope_prompt.py diff --git a/lib/resolve.py b/lib/resolve.py index ca32d1c..5832ee3 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -481,6 +481,32 @@ def fetch_linked_issue(issue_number): - Make forward progress on every turn, or call finish() to stop. """ +SCOPE = """ +## Scope: what counts as "done" + +Read the issue body AND the issue/PR comments before forming your plan. The +comments are not optional context — they often contain decisive scope +information that supersedes the body. + +**If the comments contain a substantial design analysis, implementation +spec, or output from a prior `/agent-design`, `/agent-workshop`, or +`/agent-delegate` run, treat the most recent revised version as the +binding contract for your work — NOT the issue body alone.** The issue +body may be a sketch; the spec is the contract. Implement every component +the spec lists (every file, route, table, function, test). Do NOT call +`finish(success=True)` while spec items remain unaddressed unless you have +a concrete reason a specific item should be omitted, which you post as a +comment on the issue first. + +If no design or spec comments exist, the issue body is the contract. + +When a detailed spec is present, expect a multi-file, multi-component +implementation that takes far more iterations than a typical bug fix. +Scope-reducing to "the foundational piece" and shipping a tiny PR is the +wrong outcome — it leaves the issue effectively unresolved and the user +has to re-invoke the agent on the same scope. +""" + WORKFLOW = """ ## Problem-Solving Workflow @@ -488,7 +514,7 @@ def fetch_linked_issue(issue_number): 1. **Read only what you need**: Read only the files you are about to change — no speculative exploration. If the issue already identifies the relevant files, go straight to them. You should be writing or modifying a file by iteration 5 for a simple fix, or iteration 10 for a complex multi-file change. If you are still only reading files past iteration 10, stop and start implementing. 2. **Plan**: Identify the minimal set of changes needed. -3. **Implement**: Make focused, minimal changes. Modify existing files rather than creating new ones. Never create multiple versions of the same file (e.g., fix.py alongside fix_v2.py). +3. **Implement**: Make focused changes scoped to the task. For a bug fix, prefer modifying existing files over creating new ones — and never create multiple versions of the same file (e.g., fix.py alongside fix_v2.py). For a spec-driven implementation, create exactly the new files the spec lists (no more, no fewer). 4. **Verify**: Run tests if they exist. Check that the code is syntactically valid. If tests require dependencies that aren't installed, install them first (`pip install pytest`, `npm install`, etc.) — you are allowed to install packages freely. 5. **Commit and push**: Stage all changes, commit with a clear message, and push. 6. **Finish**: Call finish() with a meaningful pr_title and pr_body. @@ -518,10 +544,16 @@ def _budget_paragraph(max_iterations): return ( f"\n## Iteration Budget\n\n" f"You have a budget of **{max_iterations} iterations** for this task. " - f"Aim to finish in significantly fewer if the task allows — the budget is a " - f"ceiling, not a target. Don't pad with extra exploration just because the " - f"budget is there. A simple fix should take 5-10 iterations; a complex " - f"multi-file change rarely needs more than 20-30.\n" + f"Aim to finish in significantly fewer if the task allows — the budget " + f"is a ceiling, not a target. Don't pad with extra exploration just " + f"because the budget is there.\n\n" + f"Typical iteration counts:\n" + f"- Simple fix (one file, focused change): 5-10 iterations.\n" + f"- Complex multi-file change: 20-30 iterations.\n" + f"- Implementing a detailed multi-component spec (see ## Scope above): " + f"50-100+ iterations — every file, route, table, and test the spec " + f"lists has to be created. Stopping at 20-30 to ship a 'foundational' " + f"subset is the wrong call when a spec is the contract.\n" ) STUCK_RECOVERY = """ @@ -644,6 +676,7 @@ def build_system_prompt(repo_context, issue_context_str, distillation_ran=False) prompt = ( AGENT_ROLE + + SCOPE + _budget_paragraph(MAX_ITERATIONS) + f"\n# Repository Context\n\n{repo_context}\n\n" + WORKFLOW diff --git a/tests/test_resolve_scope_prompt.py b/tests/test_resolve_scope_prompt.py new file mode 100644 index 0000000..18d8851 --- /dev/null +++ b/tests/test_resolve_scope_prompt.py @@ -0,0 +1,96 @@ +"""Tests for the SCOPE section of resolve.py's system prompt. + +The SCOPE section tells the agent to treat substantial design / spec +comments as the binding contract for the work, rather than scope-reducing +based on the issue body alone. This guards against the failure mode where +an agent with a vague issue body + detailed spec in comments scope-reduces +to a "foundational subset" and ships a tiny PR — see bridge-analysis #436 +(PR #437: 3 files / 40 LOC out of a ~10-file web app spec). + +The fix lives in resolve's general prompt so it works for both manual +flows (user runs /agent-design, then /agent-resolve on the same issue) +and delegate (which is conceptually the macro expansion of the manual +flow). No delegate-specific Stage 4 prompt — single source of truth. +""" + +import os +import sys +from pathlib import Path +from unittest.mock import patch + +import pytest + +REPO_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +_ENV_PATCH = patch.dict( + os.environ, + { + "ISSUE_NUMBER": "456", + "GITHUB_REPOSITORY": "owner/repo", + "LLM_MODEL": "anthropic/claude-3-5-sonnet-20241022", + "BASH_OUTPUT_LIMIT": "0", + "CONTEXT_KEEP_TOOL_RESULTS": "0", + "MAX_CONTEXT_TOKENS": "0", + "COMPACTION_COVERAGE": "0.5", + "COMPACTION_FACTOR": "0.5", + }, +) +_ENV_PATCH.start() +import lib.resolve as resolve_mod # noqa: E402 (after env patch) +_ENV_PATCH.stop() + + +class TestScopeSection: + """The SCOPE section must be present and assert the spec-as-contract rule.""" + + def test_scope_section_exists(self): + assert "## Scope: what counts as \"done\"" in resolve_mod.SCOPE + + def test_scope_mentions_design_and_spec_comments(self): + import re + # Whitespace-tolerant match (the SCOPE prompt is hard-wrapped). + normalized = re.sub(r"\s+", " ", resolve_mod.SCOPE.lower()) + assert "design analysis" in normalized + assert "implementation spec" in normalized + + def test_scope_names_the_macro_commands(self): + for cmd in ("/agent-design", "/agent-workshop", "/agent-delegate"): + assert cmd in resolve_mod.SCOPE, f"SCOPE should mention {cmd!r}" + + def test_scope_warns_against_scope_reduction(self): + # The phrase doesn't have to be verbatim, but the anti-shortcut + # framing must be present in some form so the model can pattern-match. + text = resolve_mod.SCOPE.lower() + assert "scope-reduc" in text or "shortcut" in text or "foundational" in text + + def test_scope_says_dont_finish_with_unaddressed_items(self): + text = resolve_mod.SCOPE.lower() + assert "finish" in text and "unaddressed" in text + + def test_scope_is_in_assembled_system_prompt(self): + """SCOPE must actually be wired into build_system_prompt's output.""" + prompt = resolve_mod.build_system_prompt( + repo_context="some repo context", + issue_context_str="some issue body", + ) + assert "## Scope: what counts as \"done\"" in prompt + + +class TestBudgetParagraphAcknowledgesLargerScopes: + """The iteration-budget paragraph must acknowledge spec-driven runs need + way more iterations than typical bug fixes — otherwise the old "20-30 + iterations is unusual" guidance keeps biasing the agent toward early + finish() even with SCOPE present.""" + + def test_budget_mentions_spec_driven_iteration_range(self): + text = resolve_mod._budget_paragraph(150) + assert "spec" in text.lower() + # 50+ iterations should be acknowledged as normal for spec runs. + assert "50" in text or "100" in text + + def test_budget_still_warns_against_padding(self): + # SCOPE-aware doesn't mean we lift the "don't pad" guard for small fixes. + text = resolve_mod._budget_paragraph(50) + assert "ceiling, not a target" in text + assert "pad" in text.lower() From cc723dee8cd7496c65aa57e569df676c552e0f67 Mon Sep 17 00:00:00 2001 From: Greg Novak Date: Wed, 27 May 2026 15:52:41 +0000 Subject: [PATCH 09/39] Wire /agent-review council=true into the review job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The parser already enabled council mode for /agent-review (lib/config.py:593 treats council=true as setting council_models output for review mode), but the `review:` workflow job had no council branch — it unconditionally ran the single-reviewer agentic loop. /agent-review with council=true parsed fine but produced exactly one review. Found on bridge-analysis PR #438 (issue #436): user invoked /agent-review council = true Got one review from claude-small only. ## Wiring - New "Run council code review" step in the review job, guarded `if: council_models != '[]' && council_models != ''`. Calls run_build_council (already used by build Stage 2 and delegate Stage 5) with /agent-review-flavored labels. - Existing "Run review loop" and "Post review comment" steps get the opposite guard so they only run in single-reviewer mode. - "Post cost comment (fallback)" stays unguarded; the new council step writes /tmp/llm_usage.json the same way single-reviewer does, so the fallback picks up cost in both modes. ## Label parameterization on run_build_council The function previously hard-coded "Build Stage 2 — Council Code Review" and "/agent-build Stage 2" in its banner / per-reviewer attribution / completion message. Build Stage 2 keeps the defaults. Delegate Stage 5 now passes explicit "Delegate Stage 5/6" labels (was already shipping duplicated banners — the workflow's pre-banner + the function's banner; this commit makes them deliberate). The new review-council path uses plain "Council Code Review" / "/agent-review council=true" labels so users see the command they actually invoked. 664 unit tests pass (was 662; +2 for label parameterization). --- .github/workflows/remote-dev-bot.yml | 113 +++++++++++++++++++++++++++ lib/workshop.py | 26 ++++-- tests/test_workshop.py | 69 ++++++++++++++++ 3 files changed, 202 insertions(+), 6 deletions(-) diff --git a/.github/workflows/remote-dev-bot.yml b/.github/workflows/remote-dev-bot.yml index 81264ff..f53b119 100644 --- a/.github/workflows/remote-dev-bot.yml +++ b/.github/workflows/remote-dev-bot.yml @@ -1570,6 +1570,10 @@ jobs: - name: Run review loop id: review + # Single-reviewer agentic path. Skipped when `council = true` is set on + # the trigger comment — in that case the next step runs parallel + # non-agentic reviewers via run_build_council instead. + if: needs.parse.outputs.council_models == '' || needs.parse.outputs.council_models == '[]' env: LLM_API_KEY: ${{ steps.apikey.outputs.key }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} @@ -1951,7 +1955,113 @@ jobs: f.write(review) PYEOF + - name: Run council code review + # Parallel multi-reviewer path, activated when `council = true` was set + # on the trigger comment. Calls the same run_build_council that build + # Stage 2 and delegate Stage 5 use; only the user-visible labels differ + # (banner_label / attribution_label / completion_label). + if: needs.parse.outputs.council_models != '' && needs.parse.outputs.council_models != '[]' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} + COUNCIL_MODELS: ${{ needs.parse.outputs.council_models }} + GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} + GITHUB_REPO: ${{ github.repository }} + run: | + date +%s > /tmp/start_time + PR_NUMBER=${{ github.event.issue.number || github.event.pull_request.number }} + python3 << PYEOF + import json + import os + import subprocess + import sys + + sys.path.insert(0, ".remote-dev-bot/lib") + from workshop import run_build_council + + github_repo = os.environ["GITHUB_REPO"] + github_token = os.environ["GH_TOKEN"] + pr_number = "${PR_NUMBER}" + + # PR title / body / diff were already written to /tmp by the + # "Fetch PR context" step earlier in this job. + def _read(path, default=""): + try: + with open(path) as f: + return f.read().strip() + except FileNotFoundError: + return default + + pr_title = _read("/tmp/pr_title.txt") + pr_body = _read("/tmp/pr_body.txt") + pr_diff = _read("/tmp/pr_diff.txt") + + # Cap diff like the single-reviewer path does, to avoid token blowups. + if len(pr_diff) > 100000: + pr_diff = pr_diff[:100000] + "\n\n... (diff truncated, showing first 100000 characters)" + + # Pull the linked issue if the "Fetch PR context" step found one. + linked = _read("/tmp/linked_issue.txt") + issue_title = "" + issue_body_text = linked # the file contains "Issue #N: title\n\nbody" + if linked: + first_line, _, rest = linked.partition("\n") + if first_line.startswith("Issue #"): + # "Issue #N: " + parts = first_line.split(":", 1) + if len(parts) == 2: + issue_title = parts[1].strip() + issue_body_text = rest.strip() + + council_models = json.loads(os.environ.get("COUNCIL_MODELS", "[]") or "[]") + council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}" + + env = {**os.environ, "GH_TOKEN": github_token} + + def post_pr_comment(body): + proc = subprocess.run( + ["gh", "pr", "comment", pr_number, + "--repo", github_repo, + "--body", body], + env=env, + capture_output=True, text=True, + ) + if proc.returncode != 0: + print(f"Warning: failed to post PR comment: {proc.stderr}", file=sys.stderr) + + build_result = run_build_council( + council_models=council_models, + issue_title=issue_title, + issue_body=issue_body_text, + pr_title=pr_title, + pr_body=pr_body, + pr_diff=pr_diff, + extra_instructions=council_extra_instructions, + post_comment_fn=post_pr_comment, + banner_label="Council Code Review", + attribution_label="/agent-review council=true", + completion_label="Council code review complete — awaiting human review", + ) + + # Write usage data so the "Post cost comment (fallback)" step + # below picks it up (the council path does not embed cost in any + # single per-reviewer comment). + usage = { + "input_tokens": build_result["total_input_tokens"], + "output_tokens": build_result["total_output_tokens"], + "cost": build_result["total_cost"], + "iterations": 0, + } + with open("/tmp/llm_usage.json", "w") as f: + json.dump(usage, f) + PYEOF + - name: Post review comment + # Single-reviewer agentic path only — paired with "Run review loop" + # above. Council mode posts each reviewer's comment directly from + # within run_build_council, so this step doesn't apply. + if: needs.parse.outputs.council_models == '' || needs.parse.outputs.council_models == '[]' env: GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} run: | @@ -3913,6 +4023,9 @@ jobs: pr_diff=pr_diff, extra_instructions=council_extra_instructions, post_comment_fn=post_pr_comment, + banner_label="Delegate Stage 5/6 — Council Code Review", + attribution_label="/agent-delegate Stage 5", + completion_label="Delegate Stage 5/6 complete — awaiting human review", ) # Merge council usage into aggregate diff --git a/lib/workshop.py b/lib/workshop.py index 1d55a4b..2f5067e 100644 --- a/lib/workshop.py +++ b/lib/workshop.py @@ -358,8 +358,17 @@ def run_build_council( pr_diff, extra_instructions="", post_comment_fn=None, + banner_label="Build Stage 2 — Council Code Review", + attribution_label="/agent-build Stage 2", + completion_label="Build Stage 2 complete — awaiting human review", ): - """Run Stage 2 council code reviews for build mode. + """Run parallel council code reviews on a PR. + + Used by: + - /agent-build Stage 2 (defaults) + - /agent-delegate Stage 5 (overrides labels for "Delegate Stage 5/6") + - /agent-review with council=true (overrides labels for plain + "Council Code Review", no build/delegate prefix) Runs each council model's review in parallel (non-agentic). Posts each review via post_comment_fn (defaults to print if None). @@ -368,6 +377,11 @@ def run_build_council( member's model-level extra_instructions (from council_model["extra_instructions"]) is appended per-reviewer. + banner_label, attribution_label, completion_label parameterize the + user-visible mode-specific text so the same parallel-review code can + serve build mode, delegate Stage 5, and /agent-review council=true + without each emitting "Build Stage 2 — ..." headers. + Returns dict with council_results, total_input_tokens, total_output_tokens, total_cost. """ @@ -382,8 +396,8 @@ def post(body): if not council_models: post( - "## 🏛️ Build Stage 2 — Council Code Review\n\n" - "⚠️ No council models configured. Skipping council code review.\n" + f"## 🏛️ {banner_label}\n\n" + f"⚠️ No council models configured. Skipping council code review.\n" ) return { "council_results": [], @@ -393,7 +407,7 @@ def post(body): } post( - f"## 🏛️ Build Stage 2 — Council Code Review\n\n" + f"## 🏛️ {banner_label}\n\n" f"Requesting code reviews from {len(council_models)} council member(s): " f"{', '.join('`' + m['alias'] + '`' for m in council_models)}...\n" ) @@ -465,12 +479,12 @@ def _run_single_code_review(council_model): f"🤖 **Council reviewer:** `{cr['model_alias']}` (`{cr['model_id']}`)\n\n" f"{cr['review']}\n\n" f"---\n" - f"_Council code review by `/agent-build` Stage 2 (`{cr['model_alias']}`)_" + f"_Council code review by `{attribution_label}` (`{cr['model_alias']}`)_" ) n = len(council_results) post( - f"## Build Stage 2 complete — awaiting human review\n\n" + f"## {completion_label}\n\n" f"{n} model(s) have posted code reviews above. Please review the feedback " f"and address any concerns before merging.\n" ) diff --git a/tests/test_workshop.py b/tests/test_workshop.py index a84bf5d..4fc261e 100644 --- a/tests/test_workshop.py +++ b/tests/test_workshop.py @@ -11,6 +11,7 @@ from workshop import ( build_council_review_prompt, resolve_council_models, + run_build_council, COUNCIL_REVIEW_SYSTEM_PROMPT, ) @@ -19,6 +20,74 @@ # resolve_council_models # --------------------------------------------------------------------------- +class TestRunBuildCouncilLabels: + """run_build_council is shared by build Stage 2, delegate Stage 5, and + /agent-review council=true. Each uses different banner/attribution/ + completion labels — verify the parameters are threaded through to the + posted comments so the labels can be customized per-caller.""" + + def _mock_review_result(self, alias="claude-small"): + return { + "review": f"## Code Review by {alias}\n\nLooks good.", + "model_alias": alias, + "model_id": "anthropic/test", + "input_tokens": 100, + "output_tokens": 50, + "cost": 0.01, + "elapsed": 1.0, + } + + def test_default_labels_match_build_stage_2(self): + from unittest.mock import patch + posted = [] + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "test-key"}), \ + patch("workshop.run_council_code_review", + return_value=self._mock_review_result()): + run_build_council( + council_models=[{"alias": "claude-small", "id": "anthropic/test"}], + issue_title="t", + issue_body="b", + pr_title="pt", + pr_body="pb", + pr_diff="diff", + post_comment_fn=posted.append, + ) + joined = "\n".join(posted) + assert "Build Stage 2 — Council Code Review" in joined + assert "/agent-build Stage 2" in joined + assert "Build Stage 2 complete" in joined + + def test_custom_labels_for_review_council(self): + """When /agent-review with council=true calls this function, labels + should NOT say 'Build Stage 2' — that text would confuse the user + about which command they invoked.""" + from unittest.mock import patch + posted = [] + with patch.dict(os.environ, {"ANTHROPIC_API_KEY": "test-key"}), \ + patch("workshop.run_council_code_review", + return_value=self._mock_review_result()): + run_build_council( + council_models=[{"alias": "claude-small", "id": "anthropic/test"}], + issue_title="t", + issue_body="b", + pr_title="pt", + pr_body="pb", + pr_diff="diff", + post_comment_fn=posted.append, + banner_label="Council Code Review", + attribution_label="/agent-review council=true", + completion_label="Council code review complete — awaiting human review", + ) + joined = "\n".join(posted) + # Custom labels present + assert "## 🏛️ Council Code Review" in joined + assert "/agent-review council=true" in joined + assert "Council code review complete — awaiting human review" in joined + # Build-mode labels NOT leaking in + assert "Build Stage 2" not in joined + assert "/agent-build" not in joined + + class TestResolveCouncilModels: """Tests for council model filtering logic.""" From 3bf1d24d8bbd9ab9014ead71930bc8093eb1f7cb Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 31 May 2026 04:39:03 +0000 Subject: [PATCH 10/39] Resolve prompt: methodology fidelity, end-to-end verification, write tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridge-analysis PR #438 surfaced four reinforcing failure modes that were not addressed by the earlier SCOPE prompt (PR #628): 1. Spec said "BT+EB ranking", agent shipped EB-shrunk averages with no Bradley-Terry, hid the deviation in a docstring. Used 65/150 iterations — budget wasn't the constraint, behavior was. 2. Agent never ran the FastAPI app. Multiple bugs that uvicorn + curl would have surfaced in 30 seconds shipped anyway: 17 wrong-signature TemplateResponse calls, missing row_factory, the precompute function never wired into the pipeline. 3. PR body said "All 315 existing tests pass" — true but useless. Zero tests for the new code. No assertion that BT+EB was actually applied. 4. The old WORKFLOW language — "read only what you need", "complex multi-file change rarely needs more than 20-30 iterations" — was added when we were dropping tool calls due to cost. With caching at ~70-80% hit rate and distillation working, that pressure is gone, and the language now actively discourages reading the 1500-line reference notebook that needs porting. ## Five new sections / changes in build_system_prompt - **METHODOLOGY_FAITHFULNESS** (new): when the spec names an algorithm or references existing code as authoritative, the implementation must reproduce it to within rounding error. If you can't, call finish(success=False) — never ship a stand-in with a TODO. Explicitly addresses the spec-internal-inconsistency case (spec says functions are in module X, they're actually in notebooks/Y) — grep for the real location, don't write a stub. - **DEVIATION_REPORTING** (new): if you simplified, stubbed, or skipped anything, the PR body's FIRST paragraph must say so, with a specific format. Buried docstring comments don't count. - **TESTS** (new): write a test for every new function/class/module. Existing tests passing only shows you didn't break what was there. Load-bearing methodology claims need named tests (e.g., test_leaderboard_matches_bt_eb_reference_within_tolerance). - **END_TO_END** (new): run the thing before finish(success=True). Server tasks: uvicorn + curl, expect 200. CLI: invoke on realistic input. Library: smoke test the new function. This catches the bug class unit tests miss — wrong-signature calls, missing config wiring, never-invoked functions. - **WORKFLOW + _budget_paragraph + EFFICIENCY** (softened): drop the small-fix bias ("rarely needs more than 20-30"), drop "read only what you're about to change", explicitly contemplate algorithm- extraction tasks where reading the full reference is the work. 29 tests pin the new sections + assert the old anti-exploration language is gone. 693 unit tests total pass. Synthesized with the bridge-analysis agent's independent postmortem of PR #438. The bridge agent also flagged upstream causes (spec hallucinated file locations; spec used `...` code placeholders) — those will be addressed in companion PRs to design_loop and workshop.py spec generation, since they're not resolve-prompt issues. --- lib/resolve.py | 163 +++++++++++++++++++++++++---- tests/test_resolve_scope_prompt.py | 152 ++++++++++++++++++++++++++- 2 files changed, 293 insertions(+), 22 deletions(-) diff --git a/lib/resolve.py b/lib/resolve.py index 5832ee3..3298294 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -507,30 +507,149 @@ def fetch_linked_issue(issue_number): has to re-invoke the agent on the same scope. """ +METHODOLOGY_FAITHFULNESS = """ +## Methodology faithfulness + +When the spec names a specific algorithm or methodology (e.g., "BT+EB +ranking", "BH-FDR correction", "softmax with temperature 1.0"), OR +references an existing implementation in the codebase as authoritative +("uses the logic in `notebooks/leaderboard.py`", "matches the behavior +of the prior `compute_score()` function"), your implementation must +reproduce that algorithm's results to within rounding error. **The +spec's methodology is the contract.** + +Do NOT silently substitute a "simpler but sensible" stand-in just +because the canonical algorithm is harder to port. If the reference +implementation is a Python notebook or other awkward source, port it +faithfully anyway — reading and porting a 1500-line notebook is +exactly the kind of multi-iteration work the budget exists for. + +If you genuinely cannot port the canonical implementation in this run +(missing dependencies, library version mismatch, etc.): +1. Call `finish(success=False)` — do NOT call `success=True` on a + simplified stand-in. Adding a TODO and shipping is the failure mode + the human reviewer will catch downstream by clicking around. Be + honest now. +2. In your `finish()` explanation, name the canonical reference (file + + function), what you tried, why you couldn't, and what's missing. +3. Do **not** ship the placeholder anyway. A user-facing application + that displays simplified statistics as if they were the canonical + analysis is worse than no application — it erodes trust in numbers + that look correct but aren't. + +If the spec is internally inconsistent — e.g., says "use functions from +`bridge_analysis.X`" but those functions actually live in `notebooks/Y` +— grep/find the real location and use that. A spec citation that +doesn't resolve to real code is a spec error, not an invitation to +write a placeholder. +""" + +DEVIATION_REPORTING = """ +## Reporting deviations from the spec + +If your implementation is in any way less than what the spec called for +— skipped components, simplified algorithms, placeholder stand-ins, an +acceptance test you couldn't write, a route that returns mock data — +state this **explicitly in the PR body's first paragraph**. Format: + +> **Known gaps from the spec:** +> - `web/queries/leaderboard.py::compute_rankings` ships an EB-shrunk +> average instead of the spec's BT+EB. See TODO at line 47. +> - No acceptance test asserting BT+EB output match. See TODO at +> `tests/test_leaderboard.py:89`. + +Buried code comments don't count. A reviewer reading the PR body should +see the gaps without having to grep for TODOs. If there are no gaps, +omit the section entirely — silence in the PR body means "the spec was +implemented faithfully." +""" + WORKFLOW = """ ## Problem-Solving Workflow Follow this process: -1. **Read only what you need**: Read only the files you are about to change — no speculative exploration. If the issue already identifies the relevant files, go straight to them. You should be writing or modifying a file by iteration 5 for a simple fix, or iteration 10 for a complex multi-file change. If you are still only reading files past iteration 10, stop and start implementing. -2. **Plan**: Identify the minimal set of changes needed. -3. **Implement**: Make focused changes scoped to the task. For a bug fix, prefer modifying existing files over creating new ones — and never create multiple versions of the same file (e.g., fix.py alongside fix_v2.py). For a spec-driven implementation, create exactly the new files the spec lists (no more, no fewer). -4. **Verify**: Run tests if they exist. Check that the code is syntactically valid. If tests require dependencies that aren't installed, install them first (`pip install pytest`, `npm install`, etc.) — you are allowed to install packages freely. -5. **Commit and push**: Stage all changes, commit with a clear message, and push. -6. **Finish**: Call finish() with a meaningful pr_title and pr_body. +1. **Read what you need to do the work correctly.** For a bug fix, that's usually a small set of files identified by the issue. For an algorithm extraction or porting task, that's the entire reference implementation — yes, read the 1500-line notebook end-to-end if that's where the canonical logic lives. Don't speculatively read unrelated code, but don't under-read either: a partial read that misses key context leads to wrong implementations, which costs more than the read would have. +2. **Plan**: Identify the changes needed to satisfy the spec. +3. **Implement**: For a bug fix, prefer modifying existing files over creating new ones — and never create multiple versions of the same file (e.g., fix.py alongside fix_v2.py). For a spec-driven implementation, create exactly the new files the spec lists (no more, no fewer). For every new function, class, or module you add, **write a test for it** — see "Tests" below. +4. **Verify**: Run all tests, including the ones you just wrote. Check that the code is syntactically valid. If tests require dependencies that aren't installed, install them first (`pip install pytest`, `npm install`, etc.) — you are allowed to install packages freely. +5. **End-to-end check**: Before committing, verify the change actually works end-to-end, not just "unit tests pass." See "End-to-end verification" below — this is mandatory, not optional. +6. **Commit and push**: Stage all changes, commit with a clear message, and push. +7. **Finish**: Call finish() with a meaningful pr_title and pr_body. If you simplified, skipped, or stubbed anything, follow the "Reporting deviations from the spec" rule above. Never add documentation files (CHANGES.md, NOTES.md, etc.) to version control unless the issue specifically asks for them. """ +TESTS = """ +## Tests + +For every new function, class, or module you add, write a test. Unit +tests for pure logic; integration tests for code that touches the +filesystem, database, or network. Existing tests passing doesn't tell +you your new code is correct — it only tells you that you didn't +break what was already there. A PR body that says "All N existing +tests pass" without naming a single new test is a red flag. + +**Load-bearing methodology claims need named tests.** If the spec says +"uses BT+EB ranking", there must be a test like +`test_leaderboard_matches_bt_eb_reference_within_tolerance` that +asserts the output matches a reference implementation. "BT+EB" without +such a test is just a label — and a label that the next refactor can +silently violate. + +If the spec lists tests to add, add exactly those (no more, no fewer +than what the spec calls out as required, plus whatever your judgment +says the new code needs). +""" + +END_TO_END = """ +## End-to-end verification + +Unit tests verify that the units behave as their authors thought they +would. They do NOT verify that the system actually works — the seams +between units, the configuration wiring, the runtime environment. + +Before calling `finish(success=True)`, run the thing: + +- **Web app / server tasks**: start the server (`uvicorn app:app`, + `flask run`, `python manage.py runserver`, etc.) and `curl` at + least one route. Expect a 200 from a healthy route or a deliberate + redirect from `/login`. A 500 or `ImportError` on startup means + ship-blocking bugs. +- **CLI tasks**: invoke the command on a realistic input. If the spec + says "the new `web-add-user` command adds an email to the + pass-list," actually run it and verify the email lands in the + database. +- **Library / function tasks**: write and run a smoke test that + exercises the new function with realistic inputs and asserts the + output is sensible. +- **Data pipeline tasks**: run the pipeline end-to-end on a small + fixture and verify the expected rows land in the expected tables. + +This catches the bug class that unit tests miss by design: wrong- +signature library calls (e.g., `TemplateResponse(name, context)` +when the current Starlette signature is `(request, name, context)`), +missing config wiring, functions that are never invoked by any +caller, schema/code mismatches, etc. + +If end-to-end fails and you can't fix it within the iteration budget, +that's a `finish(success=False)` — not a "ship it and hope unit tests +were enough." +""" + EFFICIENCY = """ ## Efficiency Each tool call costs real money. Be targeted and deliberate: -- **Don't over-explore.** If the issue + comments already identify the files and changes needed, go straight to implementing. Read only the files you are actually about to change. - **Read files once.** Don't re-read a file you already read unless it changed. -- **Call finish() as soon as the task is done.** Don't do unnecessary verification passes after a successful test run. -- **Exploration should be proportional to task complexity.** A one-line fix does not warrant reading 10 files. +- **Call finish() as soon as the task is genuinely done.** Done means the + spec is satisfied end-to-end, including the methodology faithfulness + rule above — not "the code compiles and unit tests pass." +- **Don't pad iterations for their own sake**, but don't shortcut either. + Iteration count should be whatever the task requires. A simple bug fix + is ~5-10 iterations. An algorithm extraction or multi-component + implementation is as many as it takes — that's what the budget is for. """ @@ -544,16 +663,20 @@ def _budget_paragraph(max_iterations): return ( f"\n## Iteration Budget\n\n" f"You have a budget of **{max_iterations} iterations** for this task. " - f"Aim to finish in significantly fewer if the task allows — the budget " - f"is a ceiling, not a target. Don't pad with extra exploration just " - f"because the budget is there.\n\n" - f"Typical iteration counts:\n" - f"- Simple fix (one file, focused change): 5-10 iterations.\n" - f"- Complex multi-file change: 20-30 iterations.\n" + f"The budget is a ceiling, not a target — don't pad iterations for " + f"their own sake, but don't shortcut either. Iteration count should " + f"be whatever the task actually requires:\n\n" + f"- Simple bug fix (one file, focused change): a handful of iterations.\n" + f"- Multi-file refactor: as many as it takes to do it right.\n" + f"- Algorithm extraction / porting from a reference implementation " + f"(e.g., notebook → module): expect to spend a meaningful fraction " + f"of the budget reading the reference end-to-end. That's the work.\n" f"- Implementing a detailed multi-component spec (see ## Scope above): " - f"50-100+ iterations — every file, route, table, and test the spec " - f"lists has to be created. Stopping at 20-30 to ship a 'foundational' " - f"subset is the wrong call when a spec is the contract.\n" + f"plan to use a large fraction of the budget. Every file, route, " + f"table, and test the spec lists has to be created — and faithfully, " + f"not as a stub. Stopping at 20-30 iterations to ship a 'foundational' " + f"subset, or shipping a simplified stand-in for a load-bearing " + f"algorithm (see ## Methodology faithfulness), is the wrong call.\n" ) STUCK_RECOVERY = """ @@ -677,9 +800,13 @@ def build_system_prompt(repo_context, issue_context_str, distillation_ran=False) prompt = ( AGENT_ROLE + SCOPE + + METHODOLOGY_FAITHFULNESS + + DEVIATION_REPORTING + _budget_paragraph(MAX_ITERATIONS) + f"\n# Repository Context\n\n{repo_context}\n\n" + WORKFLOW + + TESTS + + END_TO_END + READING_THE_TASK + f"# Task\n\n{issue_context_str}\n" + GIT_INSTRUCTIONS diff --git a/tests/test_resolve_scope_prompt.py b/tests/test_resolve_scope_prompt.py index 18d8851..fdd2f8f 100644 --- a/tests/test_resolve_scope_prompt.py +++ b/tests/test_resolve_scope_prompt.py @@ -83,14 +83,158 @@ class TestBudgetParagraphAcknowledgesLargerScopes: iterations is unusual" guidance keeps biasing the agent toward early finish() even with SCOPE present.""" - def test_budget_mentions_spec_driven_iteration_range(self): + def test_budget_mentions_spec_driven_scope(self): text = resolve_mod._budget_paragraph(150) + # The paragraph should explicitly contemplate spec-driven runs, + # not just typical bug fixes. assert "spec" in text.lower() - # 50+ iterations should be acknowledged as normal for spec runs. - assert "50" in text or "100" in text + + def test_budget_mentions_algorithm_extraction(self): + """Reading a reference notebook end-to-end is exactly the work the + budget exists for — the paragraph must say so, otherwise the agent + keeps shortcutting algorithm-port tasks.""" + text = resolve_mod._budget_paragraph(150) + text_lower = text.lower() + assert "extraction" in text_lower or "porting" in text_lower or "reference" in text_lower def test_budget_still_warns_against_padding(self): - # SCOPE-aware doesn't mean we lift the "don't pad" guard for small fixes. + # Spec-aware doesn't mean we lift the "don't pad" guard for small fixes. text = resolve_mod._budget_paragraph(50) assert "ceiling, not a target" in text assert "pad" in text.lower() + + +class TestMethodologyFaithfulnessSection: + """Spec-named algorithms / referenced existing implementations must be + ported faithfully, not approximated. Source: bridge-analysis PR #438 + shipped EB-shrunk averages where the spec said BT+EB — agent buried + the deviation in a docstring.""" + + def test_section_exists_in_module(self): + assert hasattr(resolve_mod, "METHODOLOGY_FAITHFULNESS") + assert "## Methodology faithfulness" in resolve_mod.METHODOLOGY_FAITHFULNESS + + def test_section_names_finish_success_false_as_escape(self): + text = resolve_mod.METHODOLOGY_FAITHFULNESS + assert "finish(success=False)" in text + + def test_section_warns_against_ship_with_todo(self): + text = resolve_mod.METHODOLOGY_FAITHFULNESS.lower() + # The "ship a stand-in + add a TODO" failure mode must be explicitly + # called out so the agent can't rationalize it as good practice. + assert "todo" in text and "shipping" in text + + def test_section_addresses_spec_internal_inconsistency(self): + """When the spec says 'use functions from module X' but they actually + live in notebooks/Y, the agent should grep for the real location, not + write a stub. This is the bridge-analysis #438 exact failure mode.""" + text = resolve_mod.METHODOLOGY_FAITHFULNESS.lower() + assert "spec is internally inconsistent" in text or "spec citation" in text + + def test_wired_into_assembled_prompt(self): + prompt = resolve_mod.build_system_prompt( + repo_context="ctx", issue_context_str="task" + ) + assert "## Methodology faithfulness" in prompt + + +class TestDeviationReportingSection: + """If the agent simplified, stubbed, or skipped, the PR body's first + paragraph must say so. Buried docstrings don't count.""" + + def test_section_exists(self): + assert hasattr(resolve_mod, "DEVIATION_REPORTING") + assert "Reporting deviations" in resolve_mod.DEVIATION_REPORTING + + def test_requires_first_paragraph_disclosure(self): + text = resolve_mod.DEVIATION_REPORTING.lower() + assert "first paragraph" in text and "pr body" in text + + def test_explicitly_rejects_buried_comments(self): + text = resolve_mod.DEVIATION_REPORTING.lower() + assert "buried" in text or "docstring" in text or "don't count" in text + + def test_wired_into_assembled_prompt(self): + prompt = resolve_mod.build_system_prompt( + repo_context="ctx", issue_context_str="task" + ) + assert "Reporting deviations" in prompt + + +class TestTestsSection: + """Spec-driven implementations must add tests for new code — existing + tests passing only shows you didn't break old stuff.""" + + def test_section_exists(self): + assert hasattr(resolve_mod, "TESTS") + assert "## Tests" in resolve_mod.TESTS + + def test_section_requires_tests_for_new_code(self): + text = resolve_mod.TESTS.lower() + # Must explicitly require a test per new function/class/module, + # not just "run existing tests." + assert "write a test" in text or "tests for it" in text + + def test_section_flags_methodology_claim_tests_as_load_bearing(self): + text = resolve_mod.TESTS.lower() + assert "methodology" in text and "tolerance" in text + + def test_wired_into_assembled_prompt(self): + prompt = resolve_mod.build_system_prompt( + repo_context="ctx", issue_context_str="task" + ) + assert "## Tests" in prompt + + +class TestEndToEndSection: + """Unit tests don't catch wrong-signature library calls, missing config + wiring, or never-invoked functions. The agent must actually run the + thing it built before declaring success.""" + + def test_section_exists(self): + assert hasattr(resolve_mod, "END_TO_END") + assert "## End-to-end verification" in resolve_mod.END_TO_END + + def test_section_calls_out_server_path(self): + text = resolve_mod.END_TO_END.lower() + # Web app / server is the case that surfaced the gap (bridge #438). + assert "server" in text or "uvicorn" in text + + def test_section_covers_cli_and_library_paths(self): + text = resolve_mod.END_TO_END.lower() + assert "cli" in text + assert "library" in text or "function" in text + + def test_section_names_smoke_test_examples(self): + text = resolve_mod.END_TO_END.lower() + # The bug class to catch: TemplateResponse signature, missing wiring, + # ImportError on startup. + assert "templateresponse" in text or "importerror" in text or "wrong-signature" in text + + def test_wired_into_assembled_prompt(self): + prompt = resolve_mod.build_system_prompt( + repo_context="ctx", issue_context_str="task" + ) + assert "## End-to-end verification" in prompt + + +class TestSoftenedExplorationLanguage: + """The 'read only what you need / 20-30 iterations max' language was + added when we were dropping tool calls due to cost. With prompt caching + + distillation that pressure is gone, and the language now actively + discourages the right behavior for extract-and-port tasks.""" + + def test_workflow_does_not_cap_iterations_at_thirty(self): + # The old "complex multi-file change rarely needs more than 20-30" + # is exactly the bias that capped bridge #438 at 65 iterations + # with a simplified BT stand-in. Must be gone. + text = resolve_mod.WORKFLOW.lower() + assert "rarely needs more than 20" not in text + assert "rarely needs more than 30" not in text + + def test_workflow_acknowledges_reference_reading(self): + """For algorithm-extraction tasks, reading the full reference + implementation is the right thing to do.""" + text = resolve_mod.WORKFLOW.lower() + # Must hint that reading large reference impls is sometimes correct. + assert "reference" in text or "1500" in text or "notebook" in text From 4e0e450f548d0a8e69d13bfe9de5fc0bc4d590ac Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 31 May 2026 04:41:50 +0000 Subject: [PATCH 11/39] Spec generation: verify file refs, ban `...` placeholders, require methodology tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bridge-analysis PR #438 postmortem identified two failure modes that originate in the implementation-spec generation prompt, not the resolve prompt: 1. The spec said the BT functions were "already present in bridge_analysis" — they were in notebooks/leaderboard.py. The implementer agent tried the obvious import, hit ImportError, and wrote a stub. Spec hallucinating the file location is what set the stub-shipping in motion. 2. The spec used a `def _run_bt_eb_for_cohort(...): ...` template with `...` as the body. Templates with placeholder bodies read as "supply something reasonable here" — exactly how the simplified EB-shrunk-average stand-in got shipped. The fix in PR #631 catches this kind of thing downstream in the resolve prompt (methodology faithfulness, end-to-end verification). This PR fixes the upstream cause: the spec doesn't generate those landmines in the first place. ## SPEC_DESIGN_SYSTEM_PROMPT additions - **Verify every file/function reference before citing it.** Use grep / read_file. If the design says "in module X" but it's actually "in notebooks/Y", write the spec with the correct path. - **Do NOT use `...` placeholders in code templates.** Either spell out the body sufficiently for mechanical fill-in, or write a structured semantic specification that names the canonical reference (path/file.py:line) plus precondition / postcondition / tolerance. - **Tests for load-bearing methodology claims.** Any "uses X algorithm" claim must come with a named acceptance test in the test strategy (e.g., test_leaderboard_matches_bt_eb_reference_within_tolerance). Without it, the methodology is a label, not a contract. ## SPEC_REVISION_SYSTEM_PROMPT additions The revised spec MUST preserve these three properties even if council feedback would weaken them. A reviewer saying "just write 'use the existing impl'" is wrong — preserve the explicit citation and the named test. ## Tests 10 new tests in tests/test_spec_generation_prompts.py pin the prompt content. 682 unit tests total pass. Synthesized with the bridge-analysis agent's postmortem. Companion PR #631 hardens resolve. Design loop prompt change still to come. --- lib/workshop.py | 54 ++++++++++++- tests/test_spec_generation_prompts.py | 104 ++++++++++++++++++++++++++ 2 files changed, 156 insertions(+), 2 deletions(-) create mode 100644 tests/test_spec_generation_prompts.py diff --git a/lib/workshop.py b/lib/workshop.py index fb9fec0..3776183 100644 --- a/lib/workshop.py +++ b/lib/workshop.py @@ -896,13 +896,47 @@ def _run_single_review(council_model): "2. **Function signatures** — new functions with their signatures, modified " "functions with before/after\n" "3. **Data structures / schemas** — any new fields, types, or config entries\n" - "4. **Test strategy** — which existing tests need updates, which new tests to add\n" + "4. **Test strategy** — which existing tests need updates, which new tests " + "to add (see 'Tests for load-bearing methodology claims' below)\n" "5. **Edge cases and error handling** — specific scenarios to handle\n" "6. **Risks** — anything that could go wrong during implementation\n\n" "Be concrete: reference specific line numbers and existing patterns when " "possible. Precision matters more than prose. Do NOT re-litigate the design — " "implement it as given. If you find something in the codebase that makes the " - "design impossible, flag it explicitly rather than silently changing direction." + "design impossible, flag it explicitly rather than silently changing direction.\n\n" + "## Verify every file/function reference before citing it\n\n" + "Before claiming that a function `foo` lives in `module/bar.py`, use grep " + "or read_file to verify it actually does. If the design says 'extract the " + "existing BT logic from `bridge_analysis`' and your grep shows the BT " + "functions are actually in `notebooks/leaderboard.py`, write the spec with " + "the correct path — do NOT preserve the design's incorrect citation. A " + "spec that says 'functions already present in module X' when they are not " + "is the worst kind of spec error: the implementer agent will try the " + "obvious import, fail, and then write a stub rather than spend iterations " + "hunting for the real location.\n\n" + "## Do NOT use `...` placeholders in code templates\n\n" + "If you show a function signature with body `...` (or `# TODO: implement` " + "or similar), the implementer agent reads that as 'fill in something " + "reasonable here.' That's how simplified stand-ins get shipped for " + "load-bearing logic. Instead:\n" + "- Either spell out the body sufficiently for mechanical fill-in (concrete " + "SQL, the actual algorithm in pseudocode, the specific transformations), OR\n" + "- Write a structured semantic specification: 'Body must implement X " + "algorithm (see canonical impl at `path/file.py:line`); precondition: A; " + "postcondition: B; tolerance: C.' Name the canonical reference explicitly " + "so the implementer agent can port it, not improvise.\n\n" + "## Tests for load-bearing methodology claims\n\n" + "Any methodology claim in the spec — 'uses BT+EB ranking', 'applies " + "BH-FDR correction', 'softmax with temperature 1.0' — must come with a " + "named acceptance test in the test strategy section. Format:\n\n" + " test_leaderboard_matches_bt_eb_reference_within_tolerance:\n" + " Calls compute_leaderboard_results on a fixture and asserts the\n" + " output rankings match the reference implementation in\n" + " notebooks/leaderboard.py:bt_mm_players (output theta values\n" + " within 1e-6).\n\n" + "Without such a test, the methodology claim is just a label, and the next " + "refactor can silently violate it. 'All existing tests pass' is necessary " + "but not sufficient — the spec must require tests that pin the new behavior." ) @@ -969,6 +1003,22 @@ def build_council_spec_review_prompt( "- Incorporates useful suggestions\n" "- Explains why you rejected any concerns you disagree with\n" "- Remains grounded in the approved design — do NOT re-open design questions\n\n" + "## Properties the revised spec MUST preserve or strengthen\n\n" + "These are non-negotiable — never weaken them based on council feedback " + "(council feedback that pushes in the opposite direction is wrong):\n\n" + "- **Every file/function reference must resolve to real code.** If a " + "citation in the original spec turns out to be wrong (e.g., 'in module X' " + "but actually 'in notebooks/Y'), fix the citation. Do not weaken it to " + "vague language like 'the existing implementation' — that just kicks the " + "hunt-for-the-real-location problem downstream.\n" + "- **No `...` placeholders in code templates.** If the original spec has " + "any, either fill them in OR replace with a structured semantic " + "specification that names the canonical reference (`path/file.py:line`) " + "and the precondition / postcondition / tolerance.\n" + "- **Load-bearing methodology claims must have named acceptance tests.** " + "Every 'uses X algorithm' claim must have a corresponding " + "`test_X_matches_reference_within_tolerance` entry in the test strategy. " + "If the original spec is missing one, add it.\n\n" "Output a complete, revised implementation spec (not just a diff from the original)." ) diff --git a/tests/test_spec_generation_prompts.py b/tests/test_spec_generation_prompts.py new file mode 100644 index 0000000..b9b73b8 --- /dev/null +++ b/tests/test_spec_generation_prompts.py @@ -0,0 +1,104 @@ +"""Tests for the spec generation + revision prompts in lib/workshop.py. + +Source: bridge-analysis PR #438 postmortem. The implementation spec +shipped with two prompt-shaped problems: + (1) hallucinated file location: "BT functions already present in + bridge_analysis" — actually in notebooks/leaderboard.py + (2) `...` placeholder in a function template body — the implementer + agent read this as "supply something reasonable" and shipped a + simplified stand-in + +Both problems live in the spec-generation prompt, not the resolve +prompt. PR #631 hardened resolve to catch this kind of thing +downstream; this file pins the upstream fix. +""" + +import os +import sys +from pathlib import Path + +import pytest + +REPO_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(REPO_ROOT / "lib")) + +from workshop import ( + SPEC_DESIGN_SYSTEM_PROMPT, + SPEC_REVISION_SYSTEM_PROMPT, +) + + +class TestSpecDesignPrompt: + """The agent that produces the implementation spec must (1) verify + every file/function citation against the real codebase, (2) not use + `...` placeholders in code templates, and (3) require named tests + for load-bearing methodology claims.""" + + def test_requires_verifying_file_references(self): + text = SPEC_DESIGN_SYSTEM_PROMPT.lower() + # Must explicitly tell the agent to grep / read_file before citing. + assert "verify" in text + # Must call out the failure mode (citation wrong → implementer + # stubs instead of hunting). + assert "stub" in text or "stand-in" in text + + def test_warns_about_specific_real_world_failure(self): + """The 'design says module X but code is in notebooks/Y' case is + the bridge-analysis PR #438 exact failure mode. Naming the case + with concrete examples (notebooks, bridge_analysis) makes the + rule easier to pattern-match against.""" + text = SPEC_DESIGN_SYSTEM_PROMPT.lower() + assert "notebooks" in text and "bridge_analysis" in text + + def test_bans_placeholder_bodies_in_code_templates(self): + text = SPEC_DESIGN_SYSTEM_PROMPT + # Must explicitly call out the `...` (or TODO) placeholder pattern + # and tell the agent NOT to use it. + assert "..." in text # the placeholder is mentioned literally + assert "Do NOT use" in text or "do not use" in text.lower() + + def test_offers_concrete_alternative_to_placeholders(self): + """Banning placeholders without saying what to do instead leaves + the agent confused. Must offer: spell out the body OR write a + structured semantic spec naming the canonical reference.""" + text = SPEC_DESIGN_SYSTEM_PROMPT.lower() + assert "semantic specification" in text or "canonical" in text + + def test_requires_named_test_for_methodology_claims(self): + text = SPEC_DESIGN_SYSTEM_PROMPT + # Must require a specific test format like + # test_X_matches_reference_within_tolerance. + assert "test_" in text and ("tolerance" in text.lower() or "reference" in text.lower()) + + def test_warns_methodology_claim_without_test_is_just_a_label(self): + text = SPEC_DESIGN_SYSTEM_PROMPT.lower() + # The "label vs contract" framing is the key insight from the + # bridge agent postmortem. + assert "label" in text or "all existing tests pass" in text + + +class TestSpecRevisionPrompt: + """The spec-revision prompt must preserve the spec-design properties + even if council feedback would weaken them (e.g., a reviewer says + 'this is too verbose, just say "use the existing impl"' — that + weakening must be refused).""" + + def test_preserves_real_file_references_rule(self): + text = SPEC_REVISION_SYSTEM_PROMPT.lower() + assert "real code" in text or "resolve to real" in text + + def test_preserves_no_placeholders_rule(self): + text = SPEC_REVISION_SYSTEM_PROMPT + assert "..." in text + assert "placeholder" in text.lower() + + def test_preserves_methodology_test_requirement(self): + text = SPEC_REVISION_SYSTEM_PROMPT.lower() + assert "test" in text and ("methodology" in text or "tolerance" in text) + + def test_explicitly_marks_these_rules_as_non_negotiable(self): + """Council feedback that pushes against these rules is wrong; + the revision must not weaken them. The prompt should say so.""" + text = SPEC_REVISION_SYSTEM_PROMPT.lower() + # Looking for "non-negotiable", "must preserve", or similar. + assert "non-negotiable" in text or "never weaken" in text or "must preserve" in text From b4e870eb2ef89f561c92f09619581d871535d0ed Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 31 May 2026 04:43:48 +0000 Subject: [PATCH 12/39] Design loop: verify file references, require acceptance tests for methodology claims MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Third PR in the trio addressing bridge-analysis PR #438's failure modes. PR #631 hardened resolve. PR #632 hardened spec generation. This PR hardens the design loop — the earliest point in the pipeline where methodology claims and file references first get written down. If the design hallucinates "the existing logic lives in module X" when it's actually in notebooks/Y, that error propagates: into the spec, into the implementation, eventually into a shipped stub. Catching it at the design stage is cheaper than catching it in resolve. ## DEFAULT_SYSTEM_PROMPT additions - **Verify every file/function reference before citing it.** Use grep or read_file. If the canonical impl lives in a notebook, say so explicitly — don't pretend it's in a clean module. - **Methodology claims need named acceptance tests.** Any "uses BT+EB" / "applies BH-FDR" claim in the design must come with a proposed acceptance test in the analysis (e.g., test_leaderboard_matches_bt_eb_reference_within_tolerance with a tolerance value). Without it, the methodology is a label not a contract, and the implementer agent can ship a stand-in that the existing test suite won't catch. ## Tests 4 new tests in TestDesignPromptFidelityRules. 676 total pass. Same diagnostic synthesis as PR #631 and #632 — derived from the bridge-analysis agent's postmortem + my analysis. The three together address the failure mode at every layer it could be caught: design generation (this PR), spec generation (#632), implementation (#631). --- lib/design_loop.py | 23 ++++++++++++++++++++++- tests/test_design_loop.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/lib/design_loop.py b/lib/design_loop.py index 44b2a90..1d5e663 100644 --- a/lib/design_loop.py +++ b/lib/design_loop.py @@ -206,7 +206,28 @@ def execute_tool(tool_name, arguments): "3. **Proposed approach** with specific files and changes\n" "4. **Risks and considerations**\n" "5. **Open questions** if any\n\n" - "Calibrate your analysis to the level of abstraction signaled by the issue:\n\n" + "## Verify every file/function reference before citing it\n\n" + "If your design says 'the existing implementation in `module/X.py`' or " + "'extract the logic from `package.Y`', use grep or read_file to verify " + "the code actually lives where you say it does. A design that cites the " + "wrong path becomes a spec that cites the wrong path, becomes an " + "implementation that fails the obvious import and writes a stub. Be " + "precise: name files by their actual path, function by their actual " + "definition site. If the canonical implementation lives in a notebook " + "or other awkward location, say so explicitly — don't pretend it's in " + "a clean module if it isn't.\n\n" + "## Methodology claims need named acceptance tests\n\n" + "Any load-bearing methodology claim in your design — 'uses BT+EB', " + "'applies BH-FDR correction', 'softmax with temperature 1.0' — must " + "come with a proposed acceptance test in your analysis. Format:\n\n" + " Acceptance test: test_leaderboard_matches_bt_eb_reference_within_tolerance\n" + " Asserts that compute_rankings() output matches the reference\n" + " implementation at notebooks/leaderboard.py:bt_mm_players (theta\n" + " values within 1e-6).\n\n" + "Without that, the methodology claim is a label not a contract — and " + "the implementer agent can ship a simpler stand-in that the existing " + "test suite won't catch.\n\n" + "## Calibrate your analysis to the issue's level of abstraction\n\n" "- If the issue describes a **high-level goal** (e.g., 'add workshop mode'), " "focus on architecture: components, data flow, key interfaces. Do NOT write implementation-level code.\n" "- If the issue is an **implementation spec** (e.g., specific function signatures, config schemas), " diff --git a/tests/test_design_loop.py b/tests/test_design_loop.py index 07fa4e4..d78c6e8 100644 --- a/tests/test_design_loop.py +++ b/tests/test_design_loop.py @@ -174,6 +174,41 @@ def test_prompt_mentions_gh_tool(self): assert "gh" in DEFAULT_SYSTEM_PROMPT +class TestDesignPromptFidelityRules: + """The design loop produces what eventually becomes the binding contract + for implementation (when delegate is used, or when the user manually + runs design → resolve). If the design hallucinates file locations or + treats methodology as a label rather than a contract, those errors + propagate. Source: bridge-analysis PR #438 postmortem.""" + + def test_requires_verifying_file_references(self): + """Design must grep / read_file before citing 'the existing + implementation lives in module X' — and must write the correct + path even if the issue author said something different.""" + text = DEFAULT_SYSTEM_PROMPT.lower() + assert "verify" in text + # Concrete: the failure mode where the wrong citation results + # in a stub downstream. + assert "stub" in text + + def test_warns_about_notebook_vs_module_failure_mode(self): + """The 'don't pretend it's in a clean module if it's in a notebook' + case is the exact bridge-analysis #438 failure. Naming it concretely + makes the rule easier to pattern-match against.""" + text = DEFAULT_SYSTEM_PROMPT.lower() + assert "notebook" in text + + def test_requires_acceptance_tests_for_methodology_claims(self): + text = DEFAULT_SYSTEM_PROMPT + assert "Acceptance test" in text or "acceptance test" in text.lower() + # Should provide a concrete example so the agent has a template. + assert "test_" in text and "tolerance" in text.lower() + + def test_warns_methodology_without_test_is_just_a_label(self): + text = DEFAULT_SYSTEM_PROMPT.lower() + assert "label" in text and "contract" in text + + # --------------------------------------------------------------------------- # Distillation flag # --------------------------------------------------------------------------- From 80860d3769cb1ede968cd07959cb381bc056b439 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Mon, 15 Jun 2026 16:37:05 +0000 Subject: [PATCH 13/39] Check in the 2026-06-11 comprehensive review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sequential-subagent review of rdb done 2026-06-11 against dev @ de72856 (immediately after PRs #631-633 merged). Archived here so it's accessible from anywhere, not just one workstation, and so future reviews have a template. Layout: comprehensive-review/ README.md — index + navigator 2026-06-11/ SUMMARY.md — the synthesis / entry point findings-*.md — per-phase incremental findings (5 files) PROCESS.md — how the review was conducted AGENT-PROMPTS.md — subagent prompts used per phase Findings headline (full detail in SUMMARY.md): Tier 1 (ship-stoppers): pr_title shell injection (resolve.py:1104); extra_instructions interpolated into Python heredocs in 5 council/loop steps; install.md curls a deleted yaml template; test.yml never runs on PRs to dev. Tier 2 (real degradations): ContextWindowExceededError recovery is dead code; distillation silently broken in /agent-design (heredoc unpacks 5 values from a 6-tuple); design_max_iterations / review_max_iterations declared in config but missing from parse outputs; reconcile job missing context-config env plumbing. Highest-leverage next feature: close the council loop. PR #438 showed all 3 council reviewers caught the methodology shortcut while the pipeline self-reported ✅ complete — the council is the working defense against premature victory and it's currently advisory only. No code changes in this PR; just the review record. --- .../2026-06-11/AGENT-PROMPTS.md | 101 +++++ comprehensive-review/2026-06-11/PROCESS.md | 36 ++ comprehensive-review/2026-06-11/SUMMARY.md | 349 ++++++++++++++++++ .../2026-06-11/findings-code.md | 106 ++++++ .../2026-06-11/findings-docs.md | 93 +++++ .../2026-06-11/findings-prompts.md | 110 ++++++ .../2026-06-11/findings-usage.md | 147 ++++++++ .../2026-06-11/findings-workflows.md | 138 +++++++ comprehensive-review/README.md | 22 ++ 9 files changed, 1102 insertions(+) create mode 100644 comprehensive-review/2026-06-11/AGENT-PROMPTS.md create mode 100644 comprehensive-review/2026-06-11/PROCESS.md create mode 100644 comprehensive-review/2026-06-11/SUMMARY.md create mode 100644 comprehensive-review/2026-06-11/findings-code.md create mode 100644 comprehensive-review/2026-06-11/findings-docs.md create mode 100644 comprehensive-review/2026-06-11/findings-prompts.md create mode 100644 comprehensive-review/2026-06-11/findings-usage.md create mode 100644 comprehensive-review/2026-06-11/findings-workflows.md create mode 100644 comprehensive-review/README.md diff --git a/comprehensive-review/2026-06-11/AGENT-PROMPTS.md b/comprehensive-review/2026-06-11/AGENT-PROMPTS.md new file mode 100644 index 0000000..4b78bb9 --- /dev/null +++ b/comprehensive-review/2026-06-11/AGENT-PROMPTS.md @@ -0,0 +1,101 @@ +# Subagent prompts used for the 2026-06-11 comprehensive review + +Archived for the record and to serve as a starting template for future reviews. +The findings files written by these subagents are alongside this one. + +If reusing these prompts: each phase writes incrementally to a findings file +on disk, so a crashed run can resume — re-launch the phase's subagent with +"A previous run was interrupted. Findings so far are in `<findings-file>`. +Read it first and RESUME after the last completed unit — do not redo +completed units." prepended to the prompt. + +--- + +## Phase 1 — lib/ code review (general-purpose agent) + +Review the Python library code of remote-dev-bot at /home/novak/code/remote-dev-bot (verify you are on the dev branch with `git branch --show-current` — do NOT switch branches or modify any repo files). + +Context: rdb is a GitHub Actions tool — users comment `/agent-<verb>` on issues/PRs and an LLM agent acts (opens PRs, posts design analyses, runs multi-model review councils). lib/ holds the agent loops: resolve.py (code-writing agent), reconcile.py (rebase/conflict agent), design_loop.py (read-only exploration), workshop.py (multi-model councils + delegate pipeline), distill.py (context pre-compression), config.py (3-layer config merge + inline args), context.py (compaction/trim/retry helpers), tools.py (shared tool executors), formatting.py (cost tables), cumulative_cost.py, post_fallback_cost.py, generate_index.py. + +YOUR TASK: file-by-file review of every lib/*.py for: +- [BUG] real defects (logic errors, unhandled edge cases that can fire, broken plumbing) +- [STALE] comments/docstrings that no longer match the code or reference removed features +- [DRIFT] divergence between sibling modules that share patterns (resolve/reconcile/design_loop share: budget paragraphs, status logs, cache markers, write_usage, wrapup logic — flag where one got a fix the others didn't) +- [SMELL] silent exception swallowing that could hide real failures, dead code, unused params +- [NIT] minor (only if quick to note) + +Things that are INTENTIONAL — do not flag: +- Sibling-style imports (`from formatting import ...`) in library modules vs `from lib.X import` + sys.path bootstrap in entry points (resolve.py, reconcile.py, post_fallback_cost.py). Both deliberate; see tests/test_production_imports.py. +- maybe_distill returning a 6-tuple. +- e2e tests being shell scripts (deliberate, they cost real money). +- The dogfood gate in workflows. + +CRITICAL PROTOCOL: After EACH file you finish reviewing, IMMEDIATELY append your findings for that file to <workspace>/findings-code.md using this format: + + ## lib/<name>.py (NNN lines) + - [BUG] file:line — description + - [STALE] file:line — description + (or "- clean" if nothing found) + +Write after every file — do not batch. If findings-code.md already has entries when you start, resume after the last completed file. + +After lib/, do a LIGHT skim of tests/*.py for staleness only (test names/docstrings referencing removed behavior; do not deep-review 700 tests). Append a final "## tests/ skim" section. + +Finish with a "## Top findings" section: the 5-10 most important items, ranked. Be concise throughout — findings, not essays. Do not fix anything. + +--- + +## Phase 2 — workflows + config review (general-purpose agent) + +Review the GitHub Actions workflows and config of remote-dev-bot at /home/novak/code/remote-dev-bot (dev branch — verify with `git branch --show-current`, do not modify repo files). + +Context: same tool as Phase 1. The main reusable workflow .github/workflows/remote-dev-bot.yml is ~4400 lines with 8 jobs (parse, resolve, design, review, workshop, build, reconcile, delegate). The 8 jobs contain INTENTIONALLY duplicated inline setup blocks (token + checkouts + python) — a composite-action refactor failed for external callers (PR #611 reverted it). The duplication itself is accepted; what matters is DRIFT between the duplicated blocks. + +Files to review: .github/workflows/remote-dev-bot.yml, agent.yml, dogfood.yml, test.yml, full-test-suite.yml, e2e.yml, e2e-security.yml, release.yml; remote-dev-bot.yaml (the config); tests/e2e.sh. + +Look for: +- [DRIFT] differences between the 8 jobs' duplicated setup blocks (one got a fix others didn't) +- [DRIFT] Python heredocs inside the workflow that duplicate lib/ logic and may have drifted from it (e.g., cost-table heredocs vs lib/formatting.py) +- [STALE] comments that no longer match reality. SPECIFICALLY check remote-dev-bot.yaml's "Temporarily disabled (was N) to gather baseline data" comments on bash_output_limit / context_keep_tool_results / max_context_tokens — baseline gathering was months ago, caching+distillation have since landed; flag whether these should be re-enabled. +- [BUG] env vars consumed by lib/ but not plumbed in some job; outputs set but never read; conditions that can't fire +- [SMELL] anything else notable + +Known/already-tracked — do not flag: app-id→client-id deprecation (issue #618 filed); awk-based TIMEOUT_SECONDS (intentional, fractional support); tightened e2e assertions (recent, deliberate). + +CRITICAL PROTOCOL: append findings to <workspace>/findings-workflows.md after EACH file (and for remote-dev-bot.yml, after EACH job section). Same format as: "## <file or job>" + tagged bullets. Resume-aware: if the file already has entries, continue after the last unit. Finish with "## Top findings" (ranked). Concise. Read-only except the findings file. + +--- + +## Phase 3 — documentation review (general-purpose agent) + +Review all the documentation of remote-dev-bot at /home/novak/code/remote-dev-bot (dev branch; read-only except your findings file). + +Files: README.md, CONTRIBUTING.md, AGENTS.md, CLAUDE.md, install.md, how-it-works.md, debug.md, onboarding.md, demo.md, design-workspace.md, CHANGELOG.md, .gemini/GEMINI.md. + +For each doc, cross-check its claims against the current code (use grep/read on lib/, workflows, remote-dev-bot.yaml). Hunt for: +- [STALE] features described that no longer exist, or that changed (model IDs/aliases, verb lists missing reconcile/delegate, config knobs with wrong defaults, install steps that don't match agent.yml, references to removed compilation system, OpenHands references) +- [GAP] recently-shipped features that docs never mention (v0.9 additions: delegate, reconcile, council reviews on /agent-review, distillation, cache markers, methodology-fidelity prompts) +- [WRONG] factually incorrect statements +- [NIT] minor + +Notes: CONTRIBUTING.md's compile-system references were cleaned recently (PR #619) — verify nothing was missed rather than assuming. design-workspace.md is a working scratchpad — hold it to a lighter standard (flag only actively-misleading content). + +CRITICAL PROTOCOL: append to <workspace>/findings-docs.md after EACH doc: "## <file>" + tagged bullets (or "- clean"). Resume-aware. Finish with "## Top findings". Concise. + +--- + +## Phase 4 — real-world usage analysis (general-purpose agent) + +Analyze the real-world usage of remote-dev-bot across two repos using the gh CLI (read-only; you have gh access): gnovak/remote-dev-bot (self-dev, invoked via /dogfood-* comments) and gnovak/bridge-analysis (the real consumer, invoked via /agent-* comments). + +Goal: an evidence-based picture of how the tool performs in practice, May-June 2026. The two chronic failure modes to look for evidence of: (a) OVER-EXPLORATION — agent burns iterations/cost reading without producing; (b) PREMATURE VICTORY — agent declares success with partial work (e.g., bridge-analysis PR #437: 632-LOC stub when the spec wanted a full web app; PR #438's methodology shortcut). Recent prompt fixes (rdb PRs #628, #631-633) tried to address (b) — look for any runs AFTER ~May 31 that show whether the fixes helped or whether (a) got worse (over-correction risk). + +What to gather (SAMPLE, don't be exhaustive — budget matters): +1. Inventory: list issues/PRs with /agent or /dogfood invocations in both repos (gh search or list + grep comments). Tabulate: date, repo, verb, outcome (PR merged / PR open / failure comment / no-op). +2. For ~10-15 representative runs: pull the Agent Status Log comments and 💰 Cost tables. Record: iterations used vs budget, input/output tokens, cache savings line, distillation line, cost, LOC/$, Info/$, wall-clock. +3. Failures: find "could not fully resolve" / "exited unexpectedly" comments — categorize causes. +4. Cost trend: compare early-May costs vs late-May/June (cache marker restoration merged ~May 16, distillation fixes ~May 23) — did per-run cost or cache-savings improve? + +Avoid downloading full Actions run logs (huge) — issue/PR comments contain the status logs and cost tables, which is enough. Only pull an Actions log if a specific failure needs root-cause confirmation, max 2-3 of those. + +CRITICAL PROTOCOL: append to <workspace>/findings-usage.md INCREMENTALLY: first the inventory table (write it as soon as built), then "## Run: <repo>#<num> <verb>" mini-records one at a time as you analyze each, then final "## Patterns and summary". Resume-aware: if the file has entries, continue after the last run analyzed. Concise. diff --git a/comprehensive-review/2026-06-11/PROCESS.md b/comprehensive-review/2026-06-11/PROCESS.md new file mode 100644 index 0000000..6f9fb3b --- /dev/null +++ b/comprehensive-review/2026-06-11/PROCESS.md @@ -0,0 +1,36 @@ +# Comprehensive review (2026-06-11) — process record + +How this review was produced — archived alongside the findings as +documentation of method, in case it's useful as a template for future passes. + +Started 2026-06-11 against dev @ de72856 (PRs #631-#633 just merged). +Goal: review all code on dev, note bugs + stale comments/docs, analyze +real-world usage (rdb self-dev + bridge-analysis), recommend prompting +adjustments, propose future directions. + +Method: sequential subagents writing incremental findings to disk (per the +"Fable budget protocol" memory entry — multi-hour reviews need to survive +API blocks / usage limits, so subagents append after each unit reviewed +rather than batching at the end). Subagent prompts are in `AGENT-PROMPTS.md`. + +## Phases + +- [x] 0. Workspace + dev branch sync +- [x] 1. Code review: lib/*.py (+ tests/ skim) → `findings-code.md` (11 BUG / 8 DRIFT / 12 SMELL / 10 STALE / 19 NIT; headline: CWE handler unreachable resolve.py:1523) +- [x] 2. Workflows + config review (.github/, yaml, e2e.sh) → `findings-workflows.md` (~18 BUG / 11 DRIFT / 10 STALE / 13 SMELL; headline: design-job heredoc 5-tuple unpack of `maybe_distill` → distillation silently broken in `/agent-design`; `test.yml` never runs on PRs to dev) +- [x] 3. Documentation staleness review (all *.md) → `findings-docs.md` (7 WRONG / ~21 STALE / 9 GAP / 6 NIT; headline: `install.md` curls deleted `remote-dev-bot.yaml.template` → fresh installs 404; README cites nonexistent `SECURITY_GATE`) +- [x] 4. Real-world usage analysis (gh: rdb + bridge-analysis) → `findings-usage.md` (31 invocations May 5-29, 14 deep-dives; cache savings now 3-5× paid spend; premature victory = stopping-criterion problem, 44/150 iters on stub; council caught BT shortcut but nothing acts on findings; zero runs after May 31 → #631-633 unvalidated) +- [x] 5. Prompt analysis (main session, had history context) → `findings-prompts.md` (era 1→2 prompt swing; council-loop closure as highest-leverage; model-agnosticism intact but prompt length growing) +- [x] 6. Synthesis → `SUMMARY.md` (Opus 4.7 after Fable was blocked mid-write by a usage-policy false positive on the security-vulnerability content; security findings phrased clinically on the re-write) + +## Notes for next time + +- Phase 4 (gh API usage analysis) was the cheapest and produced the most + novel insight per dollar. Worth running first next time, not last. +- Synthesizing all security findings into one document tripped Anthropic's + usage-policy classifier on Fable 5 — Opus 4.7 in a fresh-context window + did the same synthesis fine. Future reviews: lean toward a model with + more context headroom for the synthesis stage, or split the security + section into a separate file. +- All 5 findings files survived the mid-stream API block exactly because + the protocol mandated incremental disk writes. The protocol works. diff --git a/comprehensive-review/2026-06-11/SUMMARY.md b/comprehensive-review/2026-06-11/SUMMARY.md new file mode 100644 index 0000000..b2f1d98 --- /dev/null +++ b/comprehensive-review/2026-06-11/SUMMARY.md @@ -0,0 +1,349 @@ +# remote-dev-bot — Comprehensive Review (2026-06-11) + +Written 2026-06-11 against dev @ de72856 (PRs #631-633 merged). Synthesizes +five phases of incremental findings written by sequential subagents and +preserved alongside this file: `findings-{code,workflows,docs,usage,prompts}.md`. +Numbers and file:line cites come from those files; consult them for the +long-form evidence behind each item below. + +--- + +## 1. Executive summary + +rdb has reached a **maturing product** stage. The May 2026 usage record +(31 real invocations on bridge-analysis) shows the tool dependably produces +merged PRs from natural-language issues at ~$0.50–$2 per typical run and +$5–$13 per large feature, with prompt-cache savings now 3–5× the paid spend. +The original failure mode you set out to solve — agent-as-pair-programmer +requiring synchronous attention — is solved: every May invocation except one +provider outage produced a PR; every PR except one (the open web app) merged. + +Three concerns dominate the current state: + +1. **A handful of real, exploitable defects that should be fixed before any + external adoption.** Two are user-data-driven injection vectors (one shell, + one Python); one breaks every fresh install; one silently disables + distillation for `/agent-design`; one prevents unit tests from running on + the dev-branch PRs that *are* the development model. None are theoretical; + most are one- to several-line fixes. + +2. **The product risk has shifted from "does it produce a PR" to "does the PR + contain what the spec said."** Premature-victory cases (bridge-analysis + #437 schema stub; #438 silent BT-MM → EB-shrunk-averages methodology + substitution) survived after `/agent-delegate` self-reported "✅ complete." + PR #628 fixed structural completeness; PRs #631-633 (merged May 31, no + production runs yet) target the methodology-substitution residue. The + **council layer is the current working defense** against this — and it's + open-loop, advisory only. + +3. **Doc and workflow drift has accumulated past the point where users can + self-serve.** 7 outright-wrong claims (incl. a phantom security mechanism), + ~21 stale references, 9 v0.9 features absent from places users look. + Several workflow jobs have drifted away from shared lib/ helpers and + silently regressed (design lost caching, reconcile lost config plumbing). + +The prompt stack is in good shape but unvalidated since the May 31 changes. +Recommendations are below in priority order. + +--- + +## 2. Where rdb is + +### What's working + +- **Delegation that survives walking the dog.** The model you set out for — + dictate idea on phone, agent comes back with a PR — works. 16 of 17 May + resolve runs landed a PR; the one exception was a transient provider + overload, not an agent failure. +- **PR-as-output as the human gate.** Universal: every agent PR was reviewed + before merge; the gate caught the methodology shortcut on PR #438 (via the + council pass, not human eyeballs). +- **Cache savings now dominate cost.** After the moving-tail cache marker was + restored (~May 16, rdb #621), cache savings grew from $0.36/run (May 5) to + $26/run on the May 27 delegate (5× the paid spend). The "cost of using rdb" + question is largely solved for steady-state usage. +- **Multi-model councils are doing real work.** On bridge-analysis #438 all + three council reviewers independently caught the BT vs EB methodology + substitution that the agent had buried in a docstring. The cost of three + parallel reviews (~$1) bought a quality catch that a human reviewer might + reasonably miss. +- **Model-agnosticism is intact.** Prompts are plain English. Cache markers + are provider-gated in code, not prompt text. Tool-calling goes through + litellm's normalized contract. The architecture supports model-swapping + cleanly today. + +### Where the cracks are + +- **Premature victory is the dominant residual defect.** Budgets are not the + binding constraint — bridge-analysis #436's delegate run used 44 of 150 + iterations before declaring "✅ Delegate pipeline complete" on a 632-LOC + schema stub of what should have been a full web app. The stopping criterion + is the problem, not the budget. +- **Over-exploration is real but bounded.** Roughly 2–5 of every 10–30 + iterations are "let me check the current state of <file>" with no committed + changes. Worst observed: ba#392 used ~25 of 30 iterations orienting on one + notebook. Cache eats most of the cost (~$0.5–1/run waste), so this is mild + in absolute terms — but it is a real signal that the agent's + "am-I-done-reading?" heuristic is weak. +- **Doc rot has reached install-breaking severity.** `install.md` references + a config template deleted in an unrelated commit (820f891). Every fresh + install 404s. Plus 9 v0.9 features (delegate, reconcile, council reviews, + distillation, fractional timeouts, cache markers) are barely mentioned in + user-facing docs. + +### What the workflow regressions tell us + +The `/agent-design` standalone job never adopted `lib/design_loop.py` — +the module whose docstring claims to be the extraction target for this +exact case. Consequence: every design-prompt improvement (the gh tool, +budget hint, retries, cache markers, distillation fixes) shipped only for +workshop and delegate. Standalone `/agent-design` runs at 0% cache hit on +Anthropic and pays full repo context every run. Reconcile similarly has +drifted away from shared config plumbing. + +The pattern is clear: **half-finished migrations are how rdb regresses.** +The accepted-duplication strategy in `.github/workflows/remote-dev-bot.yml` +(8 jobs, intentional setup-block duplication after PR #611's composite-action +reversion) is working for the setup blocks. It is not working for the +heredoc-vs-lib boundary — that's where every drift has landed. + +--- + +## 3. Defects worth fixing before wider use + +Ranked by combined severity × likelihood. file:line cites point to the live +defect; the findings files have full reasoning. + +### Tier 1 — ship-stoppers / security + +| # | Location | What | Fix size | +|---|---|---|---| +| T1.1 | `lib/resolve.py:1104` | LLM-authored `pr_title` interpolated into `shell=True gh pr create` with only `"` escaping; backticks and `$(...)` remain live. Issue body is the upstream source. | argv list or `shlex.quote` | +| T1.2 | `.github/workflows/remote-dev-bot.yml:1173, 1599-1605, 2018, 3374-3375, 3675-3676, 4006` | Target-repo `extra_instructions` interpolated as Python literals inside heredocs (5 of 8 jobs). Multi-line YAML block scalars (the documented format) crash with SyntaxError; arbitrary Python is executable from target-repo config. | Route through `env:` like resolve does | +| T1.3 | `install.md` Step 2.3.1 | `curl` URL to `remote-dev-bot.yaml.template` returns 404 (file deleted in 820f891). Every fresh install breaks here. | Restore template or rewrite step | +| T1.4 | `.github/workflows/test.yml:5` | `pull_request: branches: [main]` only — but the documented branch model is dev-only PRs. PRs #631-633 merged with zero CI. | Add `dev` to branches list | + +T1.1 and T1.2 are the security items. Both are reachable from data a third +party can inject (issue/PR body for T1.1; target-repo `remote-dev-bot.yaml` +for T1.2). The README claims a `SECURITY_GATE` marker and a "security +microagent" hardened prompt — neither exists in the codebase. Phantom +security mitigation is doc rot worth fixing alongside the real ones; it +sets up an external reader (or yourself in six months) for a false sense +of safety. + +### Tier 2 — real bugs that materially degrade behavior + +| # | Location | What | +|---|---|---| +| T2.1 | `lib/resolve.py:1523` | `ContextWindowExceededError` subclasses `BadRequestError`; the BadRequest handler comes first → CWE recovery is dead code. Overflowing runs die with "Bad request" instead of force-trim + graceful wrap-up. | +| T2.2 | `.github/workflows/remote-dev-bot.yml:2753` | Heredoc unpacks 5 values from `maybe_distill()` which returns 6 → ValueError swallowed → every `/agent-design` run pays full-repo cost. Workshop/delegate stage 1 unaffected (use lib/). | +| T2.3 | Parse outputs missing | `design_max_iterations` and `review_max_iterations` emitted by `config.py` but never declared as parse-job outputs. Design loops run 10 iterations (hardcoded default) instead of configured 15; design wrapup (fires at 12) can never trigger. Per-invocation `max_iterations=` overrides silently ignored for design/review. | +| T2.4 | Reconcile job env block (workflow ~2289-2307) | `BASH_OUTPUT_LIMIT`/`CONTEXT_KEEP_TOOL_RESULTS`/`MAX_CONTEXT_TOKENS`/`COMPACTION_*` consumed by lib/reconcile.py but never set in the job env → reconcile runs with truncation=8000 and cache-defeating keep_n=10 against config-mandated 0s. | +| T2.5 | `lib/resolve.py:1632`, `lib/reconcile.py:598`, `lib/design_loop.py:421` | Cache-write tokens read via `getattr(prompt_details, "cache_creation_input_tokens", 0)` — litellm's field is `cache_creation_tokens`. Always returns 0. "Tokens written to cache" never appears in status logs; cache-savings formula overstates because write overhead is never subtracted. | +| T2.6 | `lib/resolve.py:1002-1076` | resolve.py runs a stale local `trim_tool_results` that drops oldest results without the read/write-aware ordering its siblings get from `lib/context.py`. The code-writing loop — the one where protecting write results matters most — is the one missing the protection. The current test suite covers the lib/context.py version, so this is invisible to CI. | +| T2.7 | `lib/resolve.py:1657`, `lib/reconcile.py:609` | "No tool calls for 3 consecutive iterations" break writes no status file, but the post-loop logic only writes status when `loop_completed_naturally` is True → runs end with no failure explanation. | +| T2.8 | `lib/distill.py:296` | `ast.get_source_segment(source, node.args)` always returns None (ast.arguments carries no position attrs). Every structural-extract signature degrades to bare positional arg names — defaults, kwonly, `*args`, `**kwargs`, annotations all dropped. Fix: `ast.unparse(node.args)`. | + +### Tier 3 — drift between sibling loops + +Resolve/reconcile/design_loop share patterns but have drifted such that each +got a fix the others didn't: + +- **Wrapup mechanics:** resolve re-injects every iteration past threshold; + reconcile injects exactly once; design_loop injects once via flag. Three + loops, three implementations of the same nudge. +- **Transient-error retry wrapping:** design_loop uses + `completion_with_retries`; reconcile uses bare `completion()` → a single + Anthropic 529 "overloaded" kills a reconcile run that the siblings would + survive. +- **Moving-tail cache markers:** resolve and reconcile have them; + design_loop doesn't → design/workshop stage 1 on Anthropic pays full + input price every iteration. +- **Compaction logic** (`lib/context.py:330`): the boundary-cut in + `compact_messages` can split an assistant(tool_calls) message from its + role="tool" results, leaving orphan tool results that providers reject + with a 400. Affects resolve and reconcile compaction paths. + +### Tier 4 — workshop/delegate-specific + +- **Delegate Stage 3 vs Stage 3c inconsistency** (`lib/workshop.py:1379`): + if the revision contains an `/agent` command, Stage 3c nulls the artifact + but Stage 3 only suppresses posting — the contaminated revised_design + still flows to Stages 3a/4. "Blocked for safety" message is cosmetic. +- **Delegate design-stage wrapup never fires** (`workflow:3643` → + `workshop.py:1192`): `graceful_wrapup_iteration` = 40 (= 0.8 × code budget + 50) is passed unchanged into 15-iteration design stages. The + wrap-up-now nudge is unreachable in design. +- **Distillation cost dropped from workshop/delegate totals** (workshop.py + 846-854, 1200-1202, 1447-1449): only loop costs summed, not + `distill_input_tokens`/`distill_cost`. resolve.py seeds totals correctly; + workshop/delegate under-report. + +--- + +## 4. Prompt recommendations (synthesizing Phase 5) + +The full reasoning is in findings-prompts.md. Headlines: + +1. **Close the council loop.** Today: delegate Stage 5 posts council reviews, + Stage 6 posts a "revision plan," nothing acts on the findings. On bridge + #438 the council unanimously caught the BT-MM substitution and the + pipeline marked itself "✅ complete" anyway. Closing the loop (council + findings triaged into blocking vs advisory → blocking triggers one bounded + revision resolve pass) would convert ~$1 of council spend into a real + quality gate. This is a code feature, **higher leverage than any further + prompt tuning.** + +2. **Validate the May 31 prompts before further changes.** PRs #631-633 + (methodology fidelity, end-to-end verification, write tests for new code) + have zero production runs. Pre-#631 data says over-exploration was real + but cache-bounded (<$1/run waste); the new prompts explicitly encourage + large reads ("read the 1500-line reference notebook end-to-end"). Expected + cost is low but should be measured. Watch the first few delegate/resolve + runs after the next release for iteration counts and cost vs May + baselines. + +3. **Migrate `/agent-design` to lib/design_loop.py.** Precondition for any + prompt work on the design verb to actually deploy. Current state: `gh` + tool, budget paragraph, retries, cache markers, and distillation fixes + are all in the lib but not in the standalone-design heredoc. This is the + workflow's largest remaining lib-duplicate. + +4. **Don't add a Stage 4½ structural-completeness checker.** Tempting — and + it would target premature-victory cleanly — but it would conflict with + the model-agnosticism goal (it'd be a separate LLM call with its own + prompt). The council-loop closure (#1 above) covers the same ground and + exists already. + +5. **Don't re-add the "20-30 iterations is plenty" minimalism language.** + It was correct under the April cost regime (no caching, dropped tool + calls). Under May+ caching it produces premature victory. The May 31 + prompts removed it for a reason; usage data confirms the call. + +6. **Consolidation pass on the resolve prompt stack** (lower priority). + Today's resolve system prompt = AGENT_ROLE + SCOPE + + METHODOLOGY_FAITHFULNESS + DEVIATION_REPORTING + budget + repo context + + WORKFLOW + TESTS + END_TO_END + READING_THE_TASK + task + + GIT_INSTRUCTIONS + EFFICIENCY + STUCK_RECOVERY + SECURITY_RULES. Coherent + today; each future edit risks introducing contradictions between + overlapping sections. A pass merging WORKFLOW.Verify with END_TO_END, and + EFFICIENCY.completion with SCOPE.completion, would shrink tokens ~30% + and the drift surface to match. Small-model compliance (gpt-small, + gemini-small) with the current long stack is untested; would be + measurable as part of the validation in #2. + +--- + +## 5. Future directions worth considering + +Ordered by what would most expand what rdb can do for you, given your stated +goal (dictate idea → walk dog → review PR). + +### A. Voice-friendly trigger surface + +You wrote the original goal as "dictate an idea for a feature into the phone +by voice." GitHub mobile's comment field is fine for typing but bad for +dictation (autocorrect mangles slash commands; no template). One thin add +would be a `/dictate <freeform>` verb that: + +- accepts any freeform first-line (no strict syntax) +- runs `/agent-workshop` if the prose looks like a feature request, or + `/agent-resolve` if it looks like a bug report (heuristic: regex for + "bug/broken/error/crash" → resolve, else workshop) +- posts a one-line "interpreted as: /agent-workshop" comment so you can + course-correct + +Cost: small (probably an addition to the parse job and a routing branch). +Removes the cognitive load of remembering which verb to use when you're +half-asleep on a train. + +### B. Council-driven revision (the closed loop from §4.1) + +Already argued above as the highest-leverage prompt-adjacent change. +Restating here as a future feature because it requires real implementation +work, not just prompt edits. + +### C. Spec-extraction pre-step for `/agent-resolve` + +When you `/agent-resolve` on a long-running issue that has accumulated +design comments (the bridge #436 pattern), there's a SCOPE prompt that +tells the agent "treat the most recent revised version as the binding +contract." It works (per #438) but it's a prompt-level instruction the +agent might or might not follow. A pre-step that mechanically extracts +"latest spec section" → a separate prompt input would be more robust and +less model-dependent. + +### D. Per-repo "what is rdb good at" memory + +The usage data shows extreme variance in LOC/$ by task shape: 1-2k LOC/$ +on greenfield notebook generation, 9-94 LOC/$ on surgical edits. A simple +per-repo profile ("this repo's average run cost is $X for tasks of shape Y") +posted as part of the cost comment would let you calibrate which work to +delegate vs do yourself. + +### E. Status-log "I'm stuck" signal + +Failure mode (a) — over-exploration — shows up reliably in status logs as +3-5 consecutive "let me check the current state of <file>" entries with no +committed changes. A side channel that detects this pattern at runtime and +posts a single "agent appears to be re-reading; intervene if you have new +context" issue comment would let you intervene before iterations are wasted. +Costs: nothing, it's pattern-matching against existing status logs. + +### F. The dogfood gap + +Usage data shows ~zero rdb dogfood in May — self-dev moved to local Claude +Code. That's likely the right choice for short tight loops, but it means +rdb only sees one real consumer (bridge-analysis), and behavior changes +that work well there might not generalize. A second consumer repo +(perhaps a smaller throwaway, or one of your other private projects) +would give the prompt-validation work in §4.2 broader signal. + +--- + +## 6. Recommended next actions + +In order, with rough effort estimates: + +1. **Tier 1 fixes** (T1.1-T1.4): ~half a day total. Ship before any wider + adoption. T1.4 alone restores CI on dev PRs which makes everything else + safer to merge. +2. **Tier 2 fixes**, especially T2.1-T2.4: another half-day. T2.2 + (distillation broken in design) and T2.4 (reconcile config plumbing) are + silently regressing costs every run. +3. **Doc patch pass**: ~2 hours. README "Draft PR by default" claims; the + phantom `SECURITY_GATE`; `/agent-delegate` and `/agent-reconcile` in + user-facing docs; config defaults across README + debug.md. +4. **Migrate `/agent-design` job to lib/design_loop.py**: bigger, ~half a + day, but converts the largest standing lib-vs-heredoc drift into a thin + wrapper. Pays dividends on every future prompt change. +5. **Validate the May 31 prompts on bridge-analysis** before any further + prompt tuning: trigger a delegate run on an open issue, watch iteration + count and cost trend vs May baselines. +6. **Close the council loop** as the next feature: highest-leverage product + change for the residual quality risk. + +This sequence trades minimum total work for maximum reduction in the +"don't share with others yet" risk profile. + +--- + +## 7. What's not on this list + +Things the review specifically found OK or out of scope: + +- Sibling-style imports vs entry-point bootstrap pattern (deliberate; tested). +- e2e tests being shell scripts (deliberate; costs real money). +- The dogfood gate in workflows (working as intended). +- `maybe_distill` 6-tuple return (deliberate; the bug is the heredoc that + unpacks 5). +- The accepted setup-block duplication across 8 workflow jobs (working). +- CLAUDE.md, .gemini/GEMINI.md, e2e-security.yml (all clean). +- `app-id` → `client-id` GitHub-action deprecation (already tracked + in issue #618). + +--- + +*End of synthesis. Source detail in +findings-{code,workflows,docs,usage,prompts}.md alongside this file.* +</content> diff --git a/comprehensive-review/2026-06-11/findings-code.md b/comprehensive-review/2026-06-11/findings-code.md new file mode 100644 index 0000000..3cb764c --- /dev/null +++ b/comprehensive-review/2026-06-11/findings-code.md @@ -0,0 +1,106 @@ +# Comprehensive review (2026-06-11) — Phase 1: lib/ code + +## lib/formatting.py (173 lines) +- [STALE] formatting.py:97 — docstring says "the bug fixed in this commit"; "this commit" is meaningless to readers of the file — should describe the fix, not the commit +- [NIT] formatting.py:11,40 — `_fmt_tok` uses uppercase 'K' (42.5K) while `_fmt_loc` uses lowercase 'k' (1.5k); inconsistent suffix casing between adjacent helpers +- [NIT] formatting.py:16,19 — `round(v)` uses banker's rounding (round(10.5M) → "10M"); cosmetic only + +## lib/tools.py (219 lines) +- [SMELL] tools.py:41-44 — repo-bounds `abs_path.startswith(repo_root)` check is dead code: the `..`/absolute checks on line 34 already guarantee normpath stays under cwd; also classic prefix-match pattern (`/repo` matches `/repo-evil`) if it ever did fire +- [STALE] tools.py:25-27 — docstring says repo_bounds=False "instead checks that the path exists"; in fact the `..`/absolute rejection still applies in that mode too (line 34-38), so "instead" is misleading +- [NIT] tools.py:149 — `git grep -n --no-color pattern` without `-e`: a pattern starting with `-` is parsed as a flag and errors; `-e` before the pattern would fix +- [NIT] tools.py:55 — dangerous-pattern list misses `rm -fr /` variant (only `-rf` matched); best-effort blocklist, noting for completeness + +## lib/context.py (463 lines) +- [BUG] context.py:330-337 — `compact_messages` cuts post-system messages at a raw index without respecting assistant(tool_calls)/tool pairing: if the boundary lands between an assistant tool_calls message (compacted) and its role="tool" results (kept in `remaining`), the new message list has orphaned tool results with no preceding tool_use — providers reject this with a 400. Affects resolve.py and reconcile.py compaction paths. +- [BUG-latent] context.py:279 — `for tc in msg.get("tool_calls", [])` raises TypeError if a message carries `tool_calls: None` (current callers never construct that, but raw litellm dicts do); `_build_tool_call_map` and `trim_tool_results` guard with truthiness, `estimate_tokens` doesn't — use `msg.get("tool_calls") or []` +- [STALE] context.py:149 — `trim_tool_results` docstring first line still says "Remove oldest tool call/result pairs"; the implementation drops individual tool results with read/write smart ordering, not whole pairs (the body text below describes the real policy; the summary line and AGENTS.md "drops the oldest pairs" description predate the smart-ordering rewrite) +- [NIT] context.py:31-37 — `classify_provider_error` regex `"([^"]+)"` truncates provider messages containing escaped quotes; cosmetic + +## lib/config.py (771 lines) +- [BUG] config.py:714-717 — `extra_instructions` / `model_extra_instructions` are written to GITHUB_OUTPUT as bare `key=value` lines. The documented config format (remote-dev-bot.yaml:33 `extra_instructions: |`) is a multi-line block scalar; a multi-line value corrupts GITHUB_OUTPUT (needs the `key<<EOF` heredoc syntax) and fails the parse job ("Invalid format"). Any user who uncomments the shipped example with >1 line hits this. +- [STALE] config.py:57-62 — `normalize_config` docstring lists "mode.context_files: → mode.extra_files:" under "Renames performed", but the code raises ValueError on `context_files` (line 73) instead of renaming; only `additional_instructions` is actually renamed +- [SMELL] config.py:443-444 — `timeout_minutes` inline arg applied twice: already folded in at line 427 (`effective_timeout`), then re-applied at 443; redundant and makes precedence reasoning harder +- [SMELL] config.py:157 — `if not name or not value: continue` silently ignores malformed arg lines like `max_iterations =` (empty value); user gets no signal their override was dropped (contrast with the loud unknown-arg error at 160) +- [SMELL] config.py:539-543 — args are logged twice in resolve_config ("Runtime args" at 368 and "Command-line args" at 539); duplicate logging block +- [NIT] config.py:399 — agent config dict still named `oh` (OpenHands legacy) after the `openhands:` → `agent:` rename +- [NIT] config.py:222 — first-line regex makes trailing punctuation fail: `/agent-resolve.` parses command_part as "resolve." → "Unknown mode" error rather than tolerating it + +## lib/cumulative_cost.py (253 lines) +- [SMELL] cumulative_cost.py:43 — prior-cost extraction sums every `**$X.XX**` in all comments, not just those inside `### 💰 Cost` tables; any unrelated bold dollar amount in an issue/PR comment (human or agent prose) silently inflates the cumulative total. The label-based anti-double-count scheme is documented but inherently fragile. +- [SMELL] cumulative_cost.py:135 — linked-issue detection regexes the concatenation of PR body + ALL comments for `Fixes #N`; a casual "Closes #N" in any discussion comment links (and double-counts) the wrong issue. Matching only the PR body would be safer. +- [NIT] cumulative_cost.py:92-112 — all gh failures silently swallowed (`except Exception: pass`); acceptable for a cosmetic table, but a one-line stderr note would aid debugging missing cumulative rows + +## lib/post_fallback_cost.py (177 lines) +- [NIT] post_fallback_cost.py:47 — `_read_usage` catches FileNotFoundError/JSONDecodeError/ValueError but not TypeError; a malformed usage file with wrong value types (e.g. `"cost": {}`) crashes the fallback poster instead of degrading to zeros +- clean otherwise + +## lib/generate_index.py (180 lines) +- [NIT] generate_index.py:120-127 — dataclass branch shows only fields; public methods on dataclasses are omitted from the index (plain classes get methods, dataclasses don't) +- [NIT] generate_index.py:88-90 — `AsyncFunctionDef` is rendered as `def` rather than `async def` +- clean otherwise + +## lib/distill.py (650 lines) +- [BUG] distill.py:296-298 — `ast.get_source_segment(source, node.args)` ALWAYS returns None: `ast.arguments` nodes carry no position attributes (verified empirically). The "real signature" path in `_format_function_sig` is dead; every signature comes from the fallback at line 302, which emits only positional arg names — defaults, `*args`, keyword-only args, `**kwargs`, and annotations are silently dropped from every structural extract. Fix: `ast.unparse(node.args)` or get_source_segment on the FunctionDef and slice. +- [SMELL] distill.py:131-134 — glob-pattern handling for SKIP_DIRS entries starting with "*" is dead code: no entry in SKIP_DIRS (line 79) starts with "*"; the "*.egg-info" comment references a pattern that was never added +- [STALE] distill.py:151 — `_truncate_content` docstring says "Truncate content to cap characters" but it truncates to 2×half_chars (4K for source files), not to cap (50K); a 50,001-char file collapses to ~4K while a 50,000-char one is kept whole — surprising cliff worth a comment if intentional +- [SMELL] distill.py:288 — `_extract_python_signatures` returns a str normally but a LIST on the empty-parts fallback; the caller compensates with an isinstance check (line 319) — inconsistent return type +- [NIT] distill.py:337-349 — Tier-1/Tier-2 distill prompt hard-requires "COMPLETE file content verbatim", but gathered files above the cap were already truncated to ~4K (marked truncated="true"); the model is instructed to do something its input makes impossible for large files + +## lib/design_loop.py (512 lines) +- [BUG] design_loop.py:421 (also resolve.py:1632, reconcile.py:598) — cache-write tracking reads `getattr(prompt_details, "cache_creation_input_tokens", 0)`, but litellm's PromptTokensDetailsWrapper field is named `cache_creation_tokens` (verified against installed litellm 1.83.7: getattr returns MISSING). cache_creation_tokens is therefore ALWAYS 0 in all three loops — "tokens written to cache" never appears in status logs, and build_cache_savings_summary never subtracts write overhead (savings overstated). `cached_tokens` (reads) is correct. +- [DRIFT] design_loop.py:437 — serializes tool calls with deprecated pydantic v1 `tc.dict()`; resolve.py:1705 uses `tc.model_dump()`. Same operation, two idioms; dict() emits DeprecationWarning and will break on a future pydantic +- [STALE] design_loop.py:448-451 — per-tool logging has branches for "bash" and "finish" tools that don't exist in this loop's TOOLS (copied from resolve); dead branches +- [STALE] design_loop.py:312-322 — run_design_loop docstring's "Returns" list omits cache_read_tokens / cache_creation_tokens, which are in the returned dict +- [DRIFT] design_loop.py — no moving-tail cache_control marker: resolve.py:1355 and reconcile.py:483 both place the moving-tail prompt-cache marker on the last message; design_loop (and workshop Stage 1/3 through it) never sets cache_control, so multi-iteration design runs on Anthropic models pay full input price every iteration. If intentional (short loops), deserves a comment; looks like the fix landed in resolve/reconcile and never propagated. + +## lib/reconcile.py (849 lines) +- [BUG] reconcile.py:607-611,788-791 — the "No tool calls for 3 consecutive iterations" break writes NO status, but the post-loop logic assumes any non-natural loop exit "wrote a more specific status" (comment at 479-481). After this break, finish_args is None and loop_completed_naturally is False, so write_status is never called — /tmp/resolve_status.json is missing/stale and the workflow reports nothing useful. (The fix in 2faf5b0 covered exception breaks but missed this break.) +- [BUG] reconcile.py:298-300 — conflict-side explanation in RECONCILE_WORKFLOW is wrong for rebase: during `git rebase origin/BASE`, the side ABOVE `=======` (HEAD/ours) is origin/BASE and the side BELOW (theirs/incoming) is the PR branch's commit being replayed. The prompt says the INCOMING side below `=======` is "from origin/{BASE_BRANCH}" — backwards; primes the agent to misattribute intent on every conflict. +- [DRIFT] reconcile.py:546 — calls bare `completion()` with no transient-error retry; design_loop.py:404 wraps via `completion_with_retries` (and resolve.py does too). A single Anthropic "overloaded" 529 kills a reconcile run that the siblings would survive. (ServiceUnavailableError lands in the generic APIError handler → fatal break.) +- [DRIFT] reconcile.py:494 vs design_loop.py:477 — wrapup-injection condition uses `==` (exact iteration) here but `>=` in design_loop; functionally similar given the injected-once flag, but the sibling logic has quietly diverged in shape +- [SMELL] reconcile.py:384-403 — system prompt contains TWO iteration-budget sections when wrapup is enabled ("## Iteration Budget" from budget_paragraph plus "## Iteration Budget — WRAP-UP REQUIRED" from wrapup_hint), stating the budget twice with different framing +- [NIT] reconcile.py:417 — `open('/tmp/agent_start_sha','w').write(...)` without close/with-block +- [NIT] reconcile.py:764 — `last_iteration + 1` reports 1 iteration even when MAX_ITERATIONS=0 and the loop never ran + +## lib/resolve.py (2167 lines) +- [BUG] resolve.py:1523-1567 — exception handler order makes the ContextWindowExceededError recovery DEAD CODE: litellm's ContextWindowExceededError subclasses BadRequestError (verified: CWE → BadRequestError → openai.BadRequestError), and `except BadRequestError` at 1523 precedes `except ContextWindowExceededError` at 1544. A context overflow is caught by the BadRequest handler → run breaks with "Bad request at iteration N" instead of force-trimming and gracefully wrapping up. Move the CWE handler above BadRequestError. +- [BUG] resolve.py:1657-1659,1681-1682,1983-1989 — same status-file gap as reconcile: the "no tool calls for 3 consecutive iterations" break (both empty-choices and no-tool-calls variants) writes NO status, but the finish_args-is-None path only writes status when loop_completed_naturally — so /tmp/resolve_status.json is never written for this exit path. +- [BUG] resolve.py:1104-1111 — `create_pr` escapes only double quotes in pr_title, then interpolates into a shell=True command: backticks, `$(...)`, and `\` inside double quotes remain live. pr_title is LLM-authored from untrusted issue text — a prompt-injected title like `$(curl ...)` executes in the runner. Use a list argv (no shell) or shlex.quote. +- [DRIFT] resolve.py:1002-1076 — resolve has its own LOCAL `trim_tool_results`, a stale copy of the pre-smart-ordering implementation (drops oldest results regardless of read/write kind). The read/write-aware version lives in lib/context.py and is used by reconcile and design_loop. Ironically resolve — the only loop that writes code — is the one that lost the "protect write results" behavior. Local copy also lacks context.py's updated drop-policy docs. +- [DRIFT] resolve.py:1407 vs reconcile.py:494 vs design_loop.py:473 — three different wrapup-injection mechanics: resolve re-injects EVERY iteration past the threshold (no injected-once flag), reconcile injects exactly once (==), design_loop injects once via flag (>=). If resolve's repeated nagging is intentional escalation, the siblings' once-only behavior (or the divergence) deserves a comment. +- [DRIFT] reconcile.py — has no ContextWindowExceededError / context-overflow recovery at all, while resolve has (intended) trim-and-wrap-up handling; long reconciles with big conflicts can overflow and die with a generic "API error" +- [STALE] resolve.py:54 — COMMIT_TRAILER env var is read but never used anywhere (config.py:24 says commit_trailer was removed); dead env plumbing +- [SMELL] resolve.py:742 — GIT_INSTRUCTIONS is a plain (non-f) string, so the agent literally sees "Must include `Fixes #{ISSUE_NUMBER}` on its own line"; models usually interpret it, but the unformatted placeholder is fragile (the finish-tool description does it right with 'Fixes #N (where N is the issue number)') +- [NIT] resolve.py:1157 — build_cost_table fallback base ref hardcodes `origin/main` ("$(git merge-base HEAD origin/main)") instead of TARGET_BRANCH; wrong LOC metrics for dev-targeting repos when /tmp/agent_start_sha is missing +- [NIT] resolve.py:2048 — pr_title fallback "Fix for issue #N" is exactly the pattern the system prompt forbids ("Bad: Fix issue #44") +- [NIT] resolve.py:2099 — `open("/tmp/cost_embedded","w").write(...)` without close/with-block (same idiom as reconcile.py:417) + +## lib/workshop.py (1710 lines) +- [BUG] workshop.py:1379-1400 vs 1644-1649 — inconsistent /agent-command blocking in delegate: Stage 3c nulls the artifact (`revised_spec = None`) when the revision contains an /agent command, but Stage 3 only suppresses the POST — the contaminated `revised_design` is still returned, fed to Stage 3a as context, and passed to the Stage 4 resolve job. The "blocked for safety" message is cosmetic for Stage 3. +- [BUG] workshop.py — distillation cost is dropped from all workshop/delegate totals: run_design_loop returns distill_input_tokens/distill_output_tokens/distill_cost separately from input_tokens/cost, and run_workshop (846-854) and run_delegate (1200-1202, 1447-1449) sum only the loop fields. resolve.py seeds its totals with distill+linked-issue cost; workshop/delegate under-report every Stage 1/3a run's real cost. +- [DRIFT] workshop.py:752 — nested f-string `f'`{m['alias']}`'` with same-quote reuse requires Python 3.12+ (PEP 701); the three sibling call sites (459, 1264, 1515) use 3.x-safe `'`' + m['alias'] + '`'` concatenation. Works on current runners but breaks py_compile on ≤3.11 and is a copy-paste landmine. +- [SMELL] workshop.py:103-107 — `_PROVIDER_KEY_MAP` duplicates config.py's KNOWN_PROVIDERS; the AGENTS.md "adding a provider" checklist doesn't mention this third copy, so a new provider's council members would be silently skipped ("API key not configured") +- [NIT] workshop.py:1210-1220 — first early-return dict from run_delegate omits the `design_analysis` / `revision_result` / `spec_revision_result` keys that the other returns include; consumers must use .get + +## lib/feedback.py (476 lines) +- [SMELL] feedback.py:388,463 — report_problems / get_consent_prompt call get_environment_info() fresh instead of using the InstallReport's own os_info/shell/python_version fields; the dataclass's auto-collected (and caller-overridable) env fields are only used by to_dict(), so a customized report still posts the live-environment values +- [NIT] feedback.py:275,336,354 — gh subprocess calls catch CalledProcessError but not FileNotFoundError; a missing gh binary raises instead of returning the documented None/False/[] failure values +- clean otherwise + +## tests/ skim (staleness only) +- [STALE] tests cover `lib.context.trim_tool_results` (test_context.py, incl. read/write smart-ordering) but NOT resolve.py's stale local copy — the function resolve actually runs is untested, and the suite green-lights behavior resolve doesn't have +- [STALE] test_cache_distill_savings.py exercises the formatting side with synthetic usage files, so it cannot catch the upstream cache_creation_tokens-always-0 extraction bug (wrong attribute name in all three loops) +- test docstrings otherwise current: legacy-key tests (openhands:/context_files:) correctly assert the raise behavior; test_no_op/test_reconcile/test_resolve_recovery import-time-env patterns documented and accurate; no references to removed features found + +## Top findings (Phase 1) +1. [BUG] resolve.py:1523 — ContextWindowExceededError recovery is dead code: it subclasses BadRequestError, whose handler comes first. Context overflow kills runs (status "Bad request") instead of trim-and-wrap-up. One-line fix: reorder handlers. +2. [BUG] resolve.py:1632 / reconcile.py:598 / design_loop.py:421 — cache-write tokens read from a nonexistent attribute (`cache_creation_input_tokens` vs litellm's `cache_creation_tokens`); always 0, so cache savings are overstated and write costs invisible in every status log. +3. [BUG] resolve.py:1104 — pr_title shell injection: LLM-authored title interpolated into shell=True `gh pr create` with only double-quote escaping; `$(...)`/backticks execute on the runner. Reachable via prompt injection from untrusted issue text. +4. [BUG] config.py:714 — multi-line `extra_instructions` (the documented `|` block-scalar format) corrupts GITHUB_OUTPUT; the shipped config example, if uncommented, breaks the parse job. +5. [DRIFT] resolve.py:1002 — resolve runs a stale local `trim_tool_results` without the read/write-aware drop ordering that context.py provides to reconcile/design_loop; the code-writing loop is the one missing the protect-write-results fix, and tests only cover the context.py version. +6. [BUG] resolve.py:1657/reconcile.py:609 — the "no tool calls ×3" break writes no status file, and the post-loop logic assumes a status was already written; the run ends with no failure explanation. +7. [BUG] reconcile.py:298 — rebase conflict-marker sides described backwards in the system prompt (says below-`=======` comes from origin/BASE; it's the PR commit), priming the agent to misattribute intent on every conflict. +8. [BUG] workshop.py:1379 — delegate Stage 3 only blocks POSTING a design containing /agent commands; the contaminated design still flows to Stages 3a/4 (Stage 3c nulls it correctly — inconsistent). +9. [BUG] distill.py:296 — `ast.get_source_segment` on ast.arguments always returns None; every structural-extract signature silently degrades to positional arg names only (no defaults/kwonly/annotations). +10. [DRIFT] design_loop.py — no moving-tail cache_control marker (resolve/reconcile both have it); design/workshop/delegate exploration stages on Anthropic models forgo prompt caching entirely. Also workshop/delegate totals drop distillation cost. diff --git a/comprehensive-review/2026-06-11/findings-docs.md b/comprehensive-review/2026-06-11/findings-docs.md new file mode 100644 index 0000000..464912f --- /dev/null +++ b/comprehensive-review/2026-06-11/findings-docs.md @@ -0,0 +1,93 @@ +# Docs review findings (Phase 3) — dev branch, 2026-06-11 + +## README.md +- [STALE] "Security microagent" bullet (~line 396): references a `SECURITY_GATE` marker that does not exist anywhere in agent.yml or remote-dev-bot.yml (grep: zero matches outside README). No "hardened system prompt instructions" against exfiltration exist in lib/resolve.py or the workflow either. "Microagent" is OpenHands-era terminology (OpenHands removed v0.6). Three README references (lines 369, 397, 410) all point at a nonexistent marker; the real gate is the `author_association` check in agent.yml. +- [WRONG] "Opens a draft PR" in How It Works step 3 and "Agent-created branches are draft PRs by default" (Recommendations, ~line 404): default `pr_type` is `ready` (remote-dev-bot.yaml, resolve.py default "ready"). Draft is only for `on_failure: draft` partial-work PRs. +- [STALE] Per-invocation args table: `bash output limit ... default: 8000` — remote-dev-bot.yaml now sets `bash_output_limit: 0` (truncation temporarily disabled). Same stale 8000 default in "Other Configuration Options" YAML block (~line 328). +- [STALE] "Other Configuration Options": `timeout_minutes: 120 ... (default: 120)` — yaml default is 60. +- [STALE] "Other Configuration Options": `context_keep_tool_results` shown as "(default: 10)" with example value 20 — yaml now sets 0 (temporarily disabled, was 20). Internally inconsistent (10 vs 20) and both wrong vs current default 0. +- [STALE] `timeout minutes` listed as type "integer" in per-invocation args table — ALLOWED_ARGS declares it `float` (fractional timeouts shipped). +- [WRONG] "You can also override `max_iterations`, `branch`, and `context` on a per-invocation basis" (~line 336): there is no `context` arg in ALLOWED_ARGS (real names: `max_context_tokens`, `context_keep_tool_results`, ...). +- [GAP] `council = true` inline arg on `/agent-review` (v0.9 headline feature) never mentioned — the Workshop/Build council section exists but review-mode council is absent. +- [GAP] Delegate and reconcile get one table row each but no descriptive section (workshop/build get a full section). Delegate's `max_design_iterations` and `design_rounds` knobs undocumented here. +- [GAP] Distillation pre-pass (`distill_enabled: true` by default — on for every user) not mentioned anywhere in README, including "Other Configuration Options". +- [STALE] Architecture section: reusable workflow "dispatches to resolve, design, or review mode" — workflow now has resolve/build/review/reconcile/design/workshop/delegate jobs. +- [NIT] Model ID examples use `anthropic/claude-sonnet-4-5` (3 places) while the shipped alias is `claude-sonnet-4-6`. + +## install.md +- [WRONG] Step 2.3.1 is broken for every new install: tells users to `curl https://raw.githubusercontent.com/gnovak/remote-dev-bot/main/remote-dev-bot.yaml.template` — that file does not exist on main or dev (added in 33e01dd, deleted in 820f891 "Handle RateLimitError..." — looks like an accidental deletion). Live URL returns 404. Either restore the template or rewrite the step. +- [STALE] "opens a draft PR for your review" (Overview), Step 4.3 "Create a draft PR", Step 4.4 "A draft PR linked to the issue" — default `pr_type` is `ready`; successful runs open ready PRs. +- [STALE] Troubleshooting "Agent doesn't trigger": "comment starts with exactly `/agent-resolve`, `/agent-design`, or `/agent-review`" — missing workshop/build/delegate/reconcile verbs. +- Verified OK: agent.yml secret/variable names (RDB_APP_ID, RDB_APP_PRIVATE_KEY, RDB_PAT_TOKEN), explicit-secrets note matches agent.yml header comment, rocket reaction exists in workflow, lib/feedback.py API (InstallReport, add_problem, set_conversation_summary, has_problems, get_consent_prompt, report_problems) all real. + +## how-it-works.md +- [STALE] "The compiled install has no visibility requirements since the workflow is self-contained" (Visibility requirements, ~line 249) — the compilation system (scripts/compile.py, dist/) was removed; there is no compiled install. Dangling reference. +- [STALE] Target-repo table (~line 65): shim "Triggers on `/agent-resolve`, `/agent-design`, `/agent-review`, `/agent-workshop`, and `/agent-build`" — missing `/agent-delegate` and `/agent-reconcile` (shim itself matches any `/agent-` prefix, so they work). +- [GAP] "How the Pieces Connect" covers resolve/workshop/build flows but says nothing about delegate (the v0.9 6-stage pipeline) or reconcile. +- [STALE] "Supported arguments" table lists only 4 of 16 ALLOWED_ARGS — presented as the complete list; missing `council`, `bash_output_limit`, `distill_enabled`, `status_log_interval`, `max_design_iterations`, `design_rounds`, compaction knobs, etc. +- [NIT] Model-resolution example resolves claude-large to `anthropic/claude-opus-4-5` — current ID is `claude-opus-4-7`. +- [NIT] Flow diagram ends "Draft PR opened in target repo" — default pr_type is ready (body text above it correctly says "draft (or ready)"). + +## CONTRIBUTING.md +- Compile-system/OpenHands leftovers: none found (grep compile/dist/openhands = clean; PR #619 cleanup held). +- [WRONG] Config Layering: "Lists replace entirely (no concatenation)" — false for `extra_files`, which is explicitly additive across all four layers with dedup (lib/config.py ~line 522-536, and how-it-works.md documents the additive behavior). True only for other lists (e.g., `council`). Needs an "except extra_files" carve-out. +- [WRONG] Dogfood shim section claims bare `/dogfood` works ("resolve with default model") — dogfood.yml's trigger requires `startsWith('/dogfood-')` or `startsWith('/dogfood ')`, so a comment of exactly `/dogfood` never fires. (dogfood.yml's own header comment has the same wrong claim.) +- [NIT] Process observation: test.yml only triggers on PRs to **main**, but the documented branch model sends every PR to **dev** — so "unit CI green" (release step 1) never actually runs on dev PRs. Docs describe the code accurately; the gap is real either way and worth an issue. +- Verified OK: e2e.sh flags (--provider/--all-models), TestLoopPreventionRegex exists, `.remote-dev-bot/` sparse-checkout path matches workflow, release.yml exists, three-layer config + ALLOWED_ARGS reference accurate. + +## AGENTS.md +- [STALE] Project intro + "How It Works" describe only resolve/design/review ("controlled via /agent-resolve, /agent-design, and /agent-review comments"); workshop/build appear in Key Files but reconcile and delegate are absent from the overview entirely. +- [STALE] Key Files: remote-dev-bot.yml "jobs: parse, resolve, design, review, workshop, build" — actual jobs also include `reconcile` and `delegate` (8 total). +- [STALE] "Adding a provider: add API key check to all **five** 'Determine API key' steps (resolve, design, review, workshop, build)" — there are now **seven** such steps (reconcile and delegate added). Stated twice (Key patterns + Common Tasks). Following the doc would leave two modes broken for a new provider. +- [STALE] Python key-files list omits lib/reconcile.py, lib/distill.py, lib/tools.py, lib/formatting.py, lib/generate_index.py, lib/cumulative_cost.py, lib/post_fallback_cost.py (design_loop.py/workshop.py only appear later in the Codebase Index). +- [STALE] Tests list covers 6 of ~20 test files — missing test_distill, test_reconcile, test_tools, test_design_loop, test_workshop, test_no_op, test_context, test_resolve_recovery, test_cache_distill_savings, etc. +- [GAP] No description of the delegate pipeline (stages, max_design_iterations/design_rounds) or the distillation pre-pass in the Codebase Index — both are places an agent working on this repo would look. +- [NIT] test_syntax.py described as checking "lib/ and scripts/" — scripts/ no longer exists (compile-system residue); the test tolerates the missing dir, but its docstring and AGENTS.md both still reference it. + +## CLAUDE.md +- clean (two-line pointer to AGENTS.md and ~/.config/agents/AGENTS.md; both exist) + +## debug.md +- [STALE] `bash_output_limit` "Default: 8000" — remote-dev-bot.yaml now ships `0` (truncation disabled for baseline data gathering). +- [STALE] `context_keep_tool_results` "Default: 20" — yaml now ships `0` (disabled, was 20). +- [GAP] README defers "tuning and observability options" to debug.md, but debug.md omits: `distill_enabled` (on by default!), `debug_logging`, `design_context_keep_tool_results` / `review_context_keep_tool_results` (all in ALLOWED_ARGS), fractional `timeout_minutes`, and the distillation/cache-savings summaries that now appear in the status-log comment (lib/formatting.py build_distillation_summary / build_cache_savings_summary). +- Verified OK: status log mechanics (side-channel call, posted as one issue comment at end — matches resolve.py), compaction resolve-only claim (design_loop.py has trim_tool_results but no compaction), compaction defaults/0.85 trigger, smart drop order (read-only first) matches context.py behavior, GITHUB_STEP_SUMMARY table exists. +- [WRONG] Correction to "Verified OK" above: debug.md's drop-order example "read-only results (file reads, grep) are dropped first" is backwards — lib/context.py explicitly PROTECTS read_file and grep results ("reference material — protect them like write ops"); only read-only *bash* command results (ls, cat, git log) are dropped first. + +## onboarding.md +- [GAP] No exercises (or even a mention) for workshop, build, delegate, reconcile, or council review — the entire v0.9 feature set. Closing line "You've now used all the main features of remote-dev-bot" is now false; at minimum add a "there's more" pointer. +- Otherwise accurate: resolve/design/review/iteration flows, rocket reaction, cost comment, model-variant guidance all match current behavior. + +## demo.md +- [STALE] "The Three Modes" table presents resolve/design/review as the complete mode list — there are now seven modes (workshop, build, delegate, reconcile missing). Header itself ("Three Modes") is wrong. +- [GAP] No examples for the v0.9 marquee flows (delegate, council review, reconcile) — this is the pre-install showcase page, the natural place to sell them. +- Examples 1-3 reference real issues/PRs and match README's; no factual issues otherwise. + +## CHANGELOG.md +- [GAP] v0.9.0 entry omits several user-visible changes that are on dev and will ship with the tag: distillation pre-pass (`distill_enabled: true` — ON by default for all users), Anthropic prompt-cache markers + cache-savings reporting in status comments, fractional `timeout_minutes`, and the config default flips `bash_output_limit` 8000→0 and `context_keep_tool_results` 20→0 (cost-relevant for every user). Human pare-down is policy, but default-behavior changes belong in the entry. +- Verified OK: v0.9 delegate 6-stage description matches code/yaml; "council ... in parallel" is correct (ThreadPoolExecutor in lib/workshop.py — workshop, build, and delegate councils all parallel); release.yml `^## v<version>` extraction matches header format; historical entries left as history. + +## how-it-works.md (addendum) +- [WRONG] "lib/workshop.py runs each council model in sequence" — councils run in PARALLEL via ThreadPoolExecutor (workshop.py lines ~495, ~755 "Run council reviews simultaneously", ~1296). CHANGELOG v0.8/v0.9 correctly say parallel. + +## design-workspace.md (scratchpad — lighter standard) +- [STALE] "Current token-saving measures" table marks tool-result trimming (keep 20) and bash truncation (8000) as **Active** — both are now 0/disabled in remote-dev-bot.yaml (the doc's own later sections say so; the table was never updated). Misleading to a reader scanning the table. +- [STALE] Row 6 "Cache marker — single, on initial user msg — Insufficient" — the moving-tail cache_control marker has since shipped (resolve.py ~1355-1513), so the assessment is resolved, not pending. Open question #1 (do explicit markers help?) is likewise settled by the implementation. +- [NIT] Sections C and F are the same proposal (raise/remove bash_output_limit) pasted twice. + +## .gemini/GEMINI.md +- clean (one-line pointer to AGENTS.md; target exists) + +## Top findings (Phase 3) +1. [WRONG] install.md Step 2.3.1: `remote-dev-bot.yaml.template` curl URL 404s — file was deleted (820f891, apparently accidentally). Breaks every fresh install at the config step. Fix before May 12 release. +2. [STALE] README Security section cites a `SECURITY_GATE` marker and "security microagent" hardened prompt that do not exist anywhere in the code (OpenHands-era residue). A documented-but-nonexistent security mitigation is the worst kind of doc rot. +3. [WRONG] "Draft PR by default" claims in README (2x) and install.md (3x) — default pr_type is `ready`. Sets wrong user expectations about review safety. +4. [STALE] AGENTS.md provider recipe says update "all five" Determine-API-key steps; there are seven (reconcile, delegate added). Following the doc ships a provider broken in two modes. Same doc's job list also omits reconcile/delegate. +5. [STALE] Default-value rot across README + debug.md: bash_output_limit "8000" (now 0), context_keep_tool_results "10"/"20" (now 0), timeout_minutes "120" (now 60). +6. [GAP] v0.9 features invisible where users look: `council=true` review absent from README; delegate/reconcile have no prose anywhere user-facing; onboarding.md claims "all the main features" covering only 3 of 7 modes; demo.md's "The Three Modes". +7. [GAP] Distillation pre-pass is ON by default for every user yet appears in no user-facing doc (README, debug.md, how-it-works) and not in the v0.9 CHANGELOG entry; ditto the 8000→0 / 20→0 default flips. +8. [WRONG] CONTRIBUTING "Lists replace entirely (no concatenation)" contradicts code + how-it-works (extra_files is additive); and how-it-works "council in sequence" contradicts code + CHANGELOG (parallel). Docs disagree with each other, code settles both. +9. [WRONG] Bare `/dogfood` documented in CONTRIBUTING (and dogfood.yml's own header) but the trigger requires `/dogfood-` or `/dogfood ` — exact `/dogfood` never fires. +10. [WRONG] debug.md drop-order example backwards: says file reads/grep dropped first; code protects them and drops read-only bash results first. + +**Overall doc health:** Core resolve/design/review flows, security gating, auth, and dev-cycle docs are solid and verified accurate — but the doc set lags ~2 releases on config defaults and v0.9 features, with one install-breaking 404 and one phantom security claim that should block the release. Tag counts: 7 WRONG, ~21 STALE, 9 GAP, 6 NIT across 12 docs (2 clean). diff --git a/comprehensive-review/2026-06-11/findings-prompts.md b/comprehensive-review/2026-06-11/findings-prompts.md new file mode 100644 index 0000000..1f2de86 --- /dev/null +++ b/comprehensive-review/2026-06-11/findings-prompts.md @@ -0,0 +1,110 @@ +# Phase 5 — Prompt analysis + +Written by the main session (has full history context: co-authored most of the +current prompt stack across PRs #613, #628, #631-633). + +## 5.1 The exploration ↔ completion tension — history and current state + +The prompt stack has swung twice: + +1. **Era 1 (Feb-Apr): cost-driven minimalism.** "Read only the files you are + about to change", "a complex multi-file change rarely needs more than 20-30 + iterations", "call finish() as soon as the task is done." Added when tool-call + trimming and bash truncation were active and every token was paid full price. + Produced the *premature victory* failure mode (bridge-analysis PR #437: + 632-LOC stub from a full web-app spec, using only 44 of 150 iterations). + +2. **Era 2 (May 31, PRs #628/#631): completion-driven expansiveness.** SCOPE + (spec-in-comments is the binding contract), METHODOLOGY_FAITHFULNESS (port + reference implementations exactly or finish(success=False)), TESTS, + END_TO_END, softened budget language ("read the 1500-line notebook + end-to-end — that's the work"). + +**Risk: era 2 is unvalidated and may over-correct.** Usage data shows zero +production runs after May 31. Pre-#631 data says over-exploration was real but +*mild and cache-bounded* (<$1/run waste; worst case ~25/30 iterations +orienting). Post-#631 prompts explicitly encourage large reads. Cache pricing +makes reads cheap, so the expected cost of over-correction is low — but it +should be *measured*, not assumed. First few delegate/resolve runs after the +next release deserve a manual look at iterations + cost vs. the May baselines. + +**Deeper observation: both eras attack the wrong variable.** The usage data +shows stopping is a *criterion* problem, not a budget problem. Era 1 said +"stop early"; era 2 says "don't stop early"; neither gives the model a +*decision procedure* for "am I done?". The current best answer is prose +("every file, route, table, test the spec lists"). A structural answer would +be stronger — see recommendations. + +## 5.2 Model-agnosticism assessment + +Goal (per Greg): prompts should be model-agnostic so models can be swapped +freely. Current state: **mostly good, with growing pressure.** + +- ✅ All prompts are plain English; no Anthropic-specific tokens/markup. +- ✅ Tool-calling contract (finish/no_op/submit_analysis) goes through litellm's + normalized function-calling — works across the three providers. +- ✅ Cache markers are provider-gated in code (`model.startswith(("anthropic/", + "claude", "gemini/", ...))`), not baked into prompt text. +- ✅ `model_extra_instructions` config gives a per-model escape hatch. +- ⚠️ **Prompt length is the emerging risk.** resolve.py's system prompt now + stacks: AGENT_ROLE + SCOPE + METHODOLOGY_FAITHFULNESS + DEVIATION_REPORTING + + budget + repo context + WORKFLOW + TESTS + END_TO_END + READING_THE_TASK + + task + GIT_INSTRUCTIONS + EFFICIENCY + STUCK_RECOVERY + SECURITY_RULES. + Frontier models handle this; small models (gpt-small, gemini-small) follow + long multi-section instruction stacks less reliably — and those are the + models council/cheap-mode runs use. No data yet on whether small models obey + SCOPE/METHODOLOGY sections; worth a cheap A/B (run the same spec-driven issue + with gpt-small and watch for stub-shipping). +- ⚠️ Some redundancy invites drift: WORKFLOW step 4 "Verify" overlaps + END_TO_END; EFFICIENCY's "call finish() as soon as genuinely done" restates + SCOPE's completion rule. Coherent today, but each future edit risks + re-introducing contradiction. A consolidation pass (one COMPLETION section, + one VERIFICATION section) would shrink tokens and drift surface. + +## 5.3 The structural gap the prompts can't fix: nothing acts on council output + +The single most interesting usage finding: in the PR #438 run, **all three +council reviewers independently caught the BT+EB methodology swap** — and the +pipeline did nothing with that. Delegate Stage 5 posts reviews; Stage 6 posts a +"revision plan" comment; no stage *acts*. The human had to notice by clicking +around the deployed app. + +The council is currently the best defense against exactly the failure mode +that prompt-prose (#631-633) tries to prevent — but it's an open-loop defense. +Closing the loop (triage council findings → blocking vs. advisory → auto-run a +bounded revision resolve for blocking findings) would convert ~$1 of council +spend into an actual quality gate. This is a code feature, not a prompt fix, +and probably higher-leverage than any further prompt tuning. + +## 5.4 Per-verb prompt reach — major caveat from Phase 2 + +Phase 2 found that the standalone `/agent-design` workflow job runs an inline +heredoc agent loop and **never adopted lib/design_loop.py**. Consequence: the +design-prompt improvements in #633 (verify file references, acceptance tests +for methodology claims), the gh tool, budget paragraph, retries, and cache +markers do NOT apply to production /agent-design — only to workshop/delegate +Stage 1, which do use the lib. So part of the prompt stack we think is deployed +is only deployed on some paths. Migrating the design job onto lib/design_loop.py +is a precondition for the prompt work to mean anything there. + +## 5.5 Recommendations (ranked) + +1. **Close the council loop** (code, not prompt): delegate Stage 6 triages + council findings; blocking findings trigger one bounded revision pass. + Highest leverage per dollar; directly targets methodology-fidelity. +2. **Migrate the /agent-design job to lib/design_loop.py** so the #633 prompt + improvements (and caching — currently 0% hit rate there) actually deploy. +3. **Structural completion check** ("Stage 4½"): after resolve finishes a + spec-driven run, a cheap model diffs spec-items vs. PR contents and posts a + checklist; unchecked items block the "✅ complete" banner. Converts the + stopping criterion from prose to procedure. +4. **Validate era-2 prompts empirically** on the next few real runs (iterations, + cost, completeness vs. May baselines) before further prompt edits. Watch + small-model compliance specifically. +5. **Consolidation pass on resolve's prompt stack** (merge overlapping + sections; aim for ~30% token cut with zero semantic change). Bonus: smaller + prompts are more model-agnostic. +6. **Don't re-add minimalism language.** Cache pricing changed the economics; + the data shows exploration waste is bounded. If costs spike, fix with cache + coverage (reconcile job currently runs cache-defeating settings — Phase 2) + rather than with "don't read" prose. diff --git a/comprehensive-review/2026-06-11/findings-usage.md b/comprehensive-review/2026-06-11/findings-usage.md new file mode 100644 index 0000000..7edd4a7 --- /dev/null +++ b/comprehensive-review/2026-06-11/findings-usage.md @@ -0,0 +1,147 @@ +# Real-world usage analysis: remote-dev-bot, May–June 2026 + +Sources: `gh api` on issue/PR comments of gnovak/remote-dev-bot and gnovak/bridge-analysis +(all comments since 2026-05-01, fetched 2026-06-XX). Read-only on GitHub. + +## Inventory + +**gnovak/remote-dev-bot (self-dev via /dogfood-\*):** essentially ZERO dogfood usage in +May–June. All 20 May PRs (#612–#633) were authored directly by gnovak (local Claude Code +sessions), not by the bot. Only trace: issue #599 "Rdb appears broken" (updated May 17). +Dogfood-driven self-dev was an April phenomenon; in May the repo's own development moved +off the bot. All real-world usage signal comes from bridge-analysis. + +**gnovak/bridge-analysis (real consumer):** 31 invocations May 5 – May 29. Verb spelled +`/agent resolve` (space form) for most; `/agent-delegate` / `/agent-review` (hyphen) late +May. Model: `claude-small` = anthropic/claude-sonnet-4-6 for every run. **No runs at all +after May 29** — issue #441 (created Jun 11) has zero comments. So the May-31 prompt fixes +(rdb #631–633) have NO production runs after them; only #628 (May 27) saw subsequent use +(May 27–29). + +| Date | Issue | Verb | Outcome | +|---|---|---|---| +| 05-05 | #355 | resolve | PR #359 MERGED | +| 05-15 | #364 | resolve | 1st attempt → no PR (cost comment only); re-invoked 01:31 same day → PR #371 MERGED | +| 05-15 | #365 | resolve | PR #366 MERGED | +| 05-15 | #367 | resolve | PR #369 MERGED | +| 05-15 | #368 | resolve | PR #370 MERGED | +| 05-17 | #374 | resolve | PR #378 MERGED | +| 05-17 | #375 | resolve | PR #377 MERGED | +| 05-18 | #384 | resolve | PR #389 MERGED | +| 05-18 | #383 | resolve | PR #391 MERGED | +| 05-19 | #393 | workshop | apparent failure (two bare Model/cost comments, no Stage 1 output); re-run next day | +| 05-20 | #394 | resolve | PR #397 MERGED | +| 05-20 | #395 | resolve | PR #398 MERGED | +| 05-20 | #392 | resolve | PR #399 MERGED | +| 05-20 | #393 | workshop | Stage 1 + Stage 2 council complete | +| 05-21 | #405 | resolve | PR #406 MERGED | +| 05-21 | #393 | design | design comment posted | +| 05-21 | #407 | resolve | PR #410 MERGED | +| 05-22 | #415 | workshop | complete | +| 05-22 | #415 | design ×2 | design comments posted (iterative refinement w/ human feedback) | +| 05-23 | #415 | resolve | PR #417 MERGED | +| 05-23 | #393 | resolve | PR #418 MERGED | +| 05-26 | #436 | delegate | full 6-stage pipeline → PR #437 MERGED (the known premature-victory stub) | +| 05-27 | #436 | resolve | PR #438 OPEN (post-#628-fix run) | +| 05-27 | #438 | review ×2 | solo review + council review posted | +| 05-29 | #439 | workshop | complete | +| 05-29 | #440 | workshop | complete | +| 05-29 | #439 | design | design posted (post-#628) | +| 05-29 | #440 | design | design posted (post-#628) | + +Verb mix: resolve 17, workshop 5, design 6, delegate 1, review 2. +Resolve success rate: 16/17 invocations produced a merged-or-open PR (one May-15 retry). +All agent resolve PRs eventually merged except #438 (open, the web-app frontend). + +## Run: bridge-analysis#355 resolve 2026-05-05 (earliest May run, pre-cache-fix) +- → PR #359 MERGED. Cost $0.49, 10 iters, 3m11s, 330K in / 15K out, 1021 LOC (new notebook), 2.1k LOC/$, 162.4 Kbit/$. +- Status log: cache line present but small — "282K tokens read from cache, ~$0.36 saved". No distillation line (feature not shipped yet). +- Cleanest run in the corpus: 10 iterations, huge single-file output. No thrash. + +## Run: bridge-analysis#364 resolve 2026-05-15 (FAILURE + retry) +- 1st attempt: crashed at iter 1 — `litellm.InternalServerError: AnthropicError … "overloaded_error"`. Failure comment: "⚠️ Agent could not fully resolve this issue", cost table shows Time 4m47s, 1 iter, 0 tokens, $0.00. Infra/provider failure, not agent behavior. +- Retry 1h later → PR #371 MERGED: $0.35, 13 iters, 1m48s, 425K in / 5.2K out, 294 LOC. +- Only hard failure among the 17 resolve invocations. + +## Run: bridge-analysis#365 resolve 2026-05-15 +- → PR #366 MERGED. $0.51, 32 iters, 3m54s, 789K in / 9.9K out, 559 LOC, 1.1k LOC/$. +- Cache: 755K read, ~$0.43 saved. No distillation line. +- Status log shows iters 20–25 "creating the notebook… [Changes: (none) [+dirty]]" — file built via uncommitted edits, mild re-reading churn but completed fine. + +## Run: bridge-analysis#375 resolve 2026-05-17 (first runs w/ distillation + restored cache) +- → PR #377 MERGED. $1.11, 18 iters, 3m58s, 1.4M in / 15K out, 816 LOC. +- Distillation line appears: "5.5K → 1.5K tokens (~$0.03 saved)" — trivial savings on small issue threads. +- Cache: 1.2M read, ~$3.29 saved — cache savings now ~3x the run cost (formula fixed by rdb #621, May 15). +- Sister run #374 → PR #378: $1.29, 18 iters, 1.5M in, cache $3.67 saved. Note: per-run COST roughly doubled vs May-15 runs ($0.35–0.57 → $1.11–1.29) as input tokens tripled; cache absorbed most of it. + +## Run: bridge-analysis#384 resolve 2026-05-18 +- → PR #389 MERGED. $1.15, 20 iters, 1.7M in, only +31/−77 LOC (refactor) → 94 LOC/$. +- Status log iters 10 & 15 identical: "Reading the current leaderboard.py to understand the exact code structure before making changes. [Changes: (none)]" — over-exploration signature (a): repeated full-file re-reads of a large notebook before any change. + +## Run: bridge-analysis#392 resolve 2026-05-20 +- → PR #399 MERGED. $1.61, 30 iters, 4m25s, 2.9M in / 16K out, +97/−30 → 78 LOC/$. +- Cache: 2.7M read, ~$7.34 saved. Distillation: 5.6K → 972 tokens. +- Status log iters 10,15,20,25 are ALL variations of "read/check leaderboard.py to understand structure before making changes" with no committed changes — the clearest mid-May over-exploration example. ~25 of 30 iterations spent orienting on one file. + +## Run: bridge-analysis#405 resolve 2026-05-21 +- → PR #406 MERGED. $1.30, 18 iters, 1.8M in, +80 LOC across 5 files → 61 LOC/$. +- Pattern repeats: iters 10/15 "check the current state of the files". +- Note LOC/$ collapse on edit-style tasks vs greenfield notebooks (61–94 LOC/$ vs 1–2k LOC/$): cost scales with reading, not writing. + +## Run: bridge-analysis#415 resolve 2026-05-23 (tactical-advice notebook) +- → PR #417 MERGED. $2.12, 27 iters, 6m41s, 3.3M in / 22K out, 1007 LOC. Cumulative (incl. workshop+2 design passes): $5.81. +- Cache: 3.0M read, ~$8.07 saved. Distillation 5.6K → 2.2K. +- Iters 10–25 all "[Changes: (none) [+dirty]]" check-current-state loops. + +## Run: bridge-analysis#393 resolve 2026-05-23 (hierarchical EB shrinkage) +- → PR #418 MERGED. $1.40, 20 iters, 2.0M in, +73/−15. Cumulative across workshop+design+resolve: $3.06. +- This is the issue where the spec said BT+EB and the agent shipped EB-only (premature-victory example (b), methodology shortcut). Status log shows no hint of the shortcut — log lines look healthy while scope was being silently narrowed. Status logs do not surface failure mode (b). + +## Run: bridge-analysis#436 delegate 2026-05-26 (the premature-victory case) +- Invoked `/agent-delegate` with `design_rounds=2 max_iterations=150` (user raised the budget). +- Stages 1–3c (design → council → revision → spec → council → revision): 16m4s, 1.2M in / 50K out, $4.06. +- Stage 4 implementation: ~20 iterations (per distillation line "× 20 iters"), ~6 min → PR #437: 6 schema tables + 4 CLI commands + tests. PR body openly frames it: "I implemented the foundational database and CLI layer" — when the issue wanted a full web app. +- Stage 6 revision: 24 iters, $1.65, +35/−7. +- Pipeline self-reports "✅ Delegate pipeline complete" with all 6 stages checkmarked. Total: 28m45s, 44/150 iterations, 6.5M in, $8.30 agentic + feature total $17.89, Cumulative LOC/$ ~2. +- KEY EVIDENCE on (b): budget was NOT the constraint — agent used 44 of 150 iterations and stopped. Premature victory is a stopping-criterion problem, not a budget problem. Status log shows the standard "check current state" lines, nothing flagging under-delivery. + +## Run: bridge-analysis#436 resolve 2026-05-27 (first run after rdb #628 prompt fix) +- → PR #438 (OPEN): 14m27s, 69 iterations, 10M in / 51K out, $5.22, 45 files, +3286 LOC, 629 LOC/$. Cumulative for the feature: $13.16 on this thread's accounting. +- Cache: 9.7M read, ~$26.17 saved (~5× the paid cost). Distillation: 417K → 7.0K tokens, ~$9.59 saved — the delegate mega-thread is where distillation finally pays. +- Direct A/B on the same issue: pre-fix delegate produced a foundation stub; post-fix resolve produced the complete app (FastAPI + HTMX, OAuth/magic-link, precompute, Terraform/nginx/systemd). All 3 council reviewers confirm architecture matches the approved design ("perfectly aligns with the agreed-upon architecture" — gemini-small). +- Iterations 69 = largest in corpus (typical 10–30), evidence for over-correction risk (a)? Partly: status log iters 10–65 are a wall of "Let me check the current state of the repository [Changes: (none) [+dirty]]" every 5 iters — but LOC/iteration (48) is in normal range (#359: 102, #417: 37), and the deliverable was ~5× larger than any other run. Verdict: iterations scaled with scope, not runaway reading; the repetitive status lines are partly a status-summarizer artifact (uncommitted work shows as "(none) +dirty"). +- BUT failure mode (b) survived in subtler form: all 3 council reviewers independently caught that `precompute.py` claims "BT-MM fit + EB shrinkage" in docstrings/PR body while `_compute_cohort_rankings` actually implements plain EB-shrunk IMP averages, self-described in a comment as a "pragmatic approximation". Structural completeness was fixed by #628; silent methodology substitution was not — which is precisely what rdb #631 ("Resolve prompt: methodology fidelity, end-to-end verification", merged May 31) targets. #631–633 have ZERO production runs since. + +## Run: bridge-analysis#438 review 2026-05-27 (solo + council) +- Solo `/agent-review council=true` 13:41 → detailed single-model review 3 min later (read/write-DB misuse, OAuth state dict issues, next_url validation). +- Second invocation 18:07 → council of 3 posted within 2 min. Verdicts: needs-changes (gpt-small, claude-small), LGTM-minor (gemini-small). High-quality catches incl. the BT+EB mislabel and dead `ALLOWED_EMAILS` config. +- Failure artifact: trailing comment "⚠️ Agent did not complete — partial cost: 57s, 0 iterations, 99K in / 8.5K out, $0.18" posted AFTER "Council code review complete". Looks like a reporting bug in the review job's final status step, not a real failure (reviews all posted). + +## Run: bridge-analysis#439 & #440 workshop+design 2026-05-29 (post-#628 design behavior) +- Workshops: Stage 1 ~2–4 min each, Stage 2 council 3 reviewers, clean. +- Design revisions: #439 — 6 iters, 3m31s, 598K in, $1.84; #440 — 5 iters, 2m9s, 452K in, $1.40. +- Pre-#628 design baselines: #393 (May 21) 3 iters $0.75; #415 (May 22) 4 iters $1.14 and 5 iters $1.60. Post-#628 design iterations rose modestly (3–5 → 5–6), cost similarly ($0.75–1.60 → $1.40–1.84). No design-loop blowup. +- rdb issue #630 (May 29, open) filed off run #439: "the cost table doesn't have the number of iterations" — user actively dogfooding the reporting. + +## Failures: taxonomy (complete list found) +1. **Provider overload** — ba#364 1st attempt (May 15): litellm InternalServerError "overloaded_error" at iter 1, $0.00, clean failure comment with cost table; manual retry 1h later succeeded. 1 occurrence. +2. **Workshop startup crash** — ba#393 (May 19): "Workshop process exited unexpectedly", 1s wall-clock, 0/15 iterations, $0.00. Re-run next day succeeded. 1 occurrence. +3. **Phantom 'did not complete'** — ba PR#438 (May 27): "Agent did not complete — partial cost" with 0 iterations posted after a fully successful council review; reporting bug, no user-visible work lost. 1 occurrence. +4. **Silent no-op invocations** — rdb#599: four `/dogfood design` comments (Apr 29–30) produced nothing; root-caused by a later design run (workflow trigger path), marked resolved May 17. Pre-May tail. +- Notably ABSENT: no context-window blowups, no wrapup-forced incomplete PRs, no failure comments at all between May 20 and May 29 despite the heaviest usage. + +## Patterns and summary + +**Verb mix (31 invocations, ba):** resolve 17, design 6, workshop 5, review 2, delegate 1. rdb dogfood: ~0 in May (self-dev moved to local Claude Code; bot usage is now 100% bridge-analysis). + +**Typical cost per verb:** resolve $0.35–2.12 (median ~$1.1; outlier $5.22 web app); design $0.75–1.84; workshop Stage1+2 ~$1–2; council review ~$0.2–0.5/reviewer; delegate full pipeline $8.30 agentic / $17.89 feature-total. + +**Cost trend (May 5 → May 27):** per-run cost ROSE ~3–10×: $0.49 (May 5) → $0.35–0.57 (May 15) → $0.94–1.29 (May 17–18) → $1.30–2.12 (May 20–23) → $5.22 (May 27). Driver is input-token growth (330K → 10M per run) from bigger issue threads, design-context-as-binding, and repo growth — not per-token inefficiency. Meanwhile cache savings grew faster than cost: $0.36 saved (May 5) → $3–4 (May 17–18, after #621 fixed the formula/marker ~May 15–16) → $7–8 (May 20–23) → $26.17 (May 27): the cache now absorbs 3–5× the paid spend. Distillation is negligible on ordinary issues (5.6K → ~1.5K, $0.03–0.05/run) but decisive on delegate mega-threads (413–417K → 7K, $3.5–9.6/run saved) after the May 23 #626 fixes. + +**Economics by task shape:** greenfield notebook generation is spectacularly cheap (1–2k LOC/$); surgical edits to large notebooks are 10–100× worse per LOC (9–94 LOC/$) because cost scales with reading, not writing. The README-metric Info/$ tracks the same gradient (162 → 5 Kbit/$). + +**Failure mode (a) over-exploration:** pervasive in MILD form — nearly every status log shows 2–5 consecutive "check the current state / re-read the file" iterations with no committed changes (worst: #392, ~25 of 30 iterations orienting on one large notebook). But absolute waste is bounded (~$0.5–1/run) because cache eats re-reads. Post-#628 over-correction: NOT confirmed — design runs +1–2 iterations; the 69-iteration web-app run scaled with deliverable size. One caveat: only 4 post-#628 runs exist; #631–633's "read the reference implementation" prompts have zero runs. + +**Failure mode (b) premature victory:** confirmed and the dominant real problem. Delegate #436 stopped at 44/150 iterations declaring a 6-stage "✅ complete" for a foundation stub; budgets are not the binding constraint, the stopping criterion is. Post-#628 the same issue re-run delivered the full 45-file app (structural completeness FIXED), but the agent still silently downgraded the ranking methodology (claims BT+EB, implements shrunk IMP means, self-labels "pragmatic approximation") — caught by all 3 council reviewers, i.e., the council layer is currently the working defense against (b). The May 31 prompt fixes (#631 methodology fidelity, #632 spec placeholders ban, #633 design acceptance tests) target exactly this residue but have NO production evidence yet — usage stopped May 29; issue #441 (Jun 11) has had no invocation. + +**Meta-observation:** the tool's heaviest real-world month shows a maturing loop: human invokes workshop → reads council → invokes design → invokes resolve → invokes council review on the PR. Every one of the 17 resolve invocations except one infra crash ended in a PR, and every agent PR except the open #438 was merged. The product risk is no longer "does it produce PRs" but "does the PR contain what the spec said" — and the cost ceiling per feature (~$18 for the web app including all governance stages) remains far below human cost. diff --git a/comprehensive-review/2026-06-11/findings-workflows.md b/comprehensive-review/2026-06-11/findings-workflows.md new file mode 100644 index 0000000..90dd7aa --- /dev/null +++ b/comprehensive-review/2026-06-11/findings-workflows.md @@ -0,0 +1,138 @@ +# Phase 2 findings — workflows + config (.github/, remote-dev-bot.yaml, e2e.sh) + +Reviewed on dev branch. Incremental; resume after last "##" unit if interrupted. + +## remote-dev-bot.yaml + +- [STALE] **Lines 177-178 (`bash_output_limit: 0`) — "Temporarily disabled (was 8000) to gather baseline data" is no longer accurate and should be acted on.** The comment dates to 2026-04-12 (commit 3908fad). Since then: moving-tail prompt caching landed and works (3831a8f, cache hit 5-10% -> 70-80%) and distillation is on. The baseline-gathering phase is over. design-workspace.md SS C/F itself already concluded the permanent value: **re-enable at ~50,000 chars** (~12.5K tokens) as a safety net against pathological cats of huge files — generous enough to avoid the multi-`sed -n` dance that the old 8000 caused, bounded enough to stop one accidental dump blowing the context. Recommend: `bash_output_limit: 50000` + rewrite comment to cite the decision, not the experiment. +- [STALE] **Lines 186-187 (`context_keep_tool_results: 0`) — same stale "gather baseline data" rationale, but here the right move is keep 0 and rewrite the comment.** design-workspace.md's conclusion (confirmed by the cache-marker work): dropping tool pairs rewrites mid-history messages, invalidates the prompt-cache prefix, and costs more than it saves. 0 is the correct *permanent* value, but the comment says "temporarily" — a future reader (or agent) could "helpfully" restore 20 and tank the cache hit rate. Rewrite as a deliberate decision. +- [STALE] **Line 194 (`max_context_tokens: 0`) — no safety valve against context overflow.** design-workspace.md says keep compaction disabled for cost (each compaction = LLM call + cache invalidation), revisit as quality lever. But with no drops AND no truncation, a 100+ iter run on Sonnet's 200K window can genuinely overflow, and design idea A's "forced drop near model limit" was never implemented — nothing guards the hard limit; a long run dies with a context-length API error. Recommend: enable compaction as safety valve (e.g. `max_context_tokens: 180000`) or implement idea A. Also line 193 "Typical model limits: ... GPT-4o 128k" — stale model reference (config ships gpt-5.1/5.5). +- [STALE] Line 153: "'openhands:' is accepted as a backward-compatible alias for 'agent:'" — legacy OpenHands-origin alias with no BACKCOMPAT(remove-at-...) marker per repo convention; candidate for tagging or removal pre-1.0. +- [SMELL] `modes.review` is the only mode without an explicit `default_model` (inherits top-level claude-small, same behavior) — harmless inconsistency vs the other six modes. +- [SMELL] design-workspace.md ideas D ("structural extract in agent context — Decided — implement") and E ("remove git ls-files — Decided — implement") are marked decided but doc doesn't record whether they shipped; sync doc when the yaml comments get refreshed (overlaps Phase 3). + +## .github/workflows/agent.yml + +- [STALE] Line 92 comment: "Available modes: resolve (fix issue + open PR), design (analysis comment), review (review a PR)" — lists only 3 of the 8 modes (missing workshop, build, reconcile, delegate). This is the shim users copy into their repos, so the stale list is user-facing. +- [SMELL] The `gate` and `resolve` jobs duplicate the same trigger condition (comment-prefix + author_association); a change to one (e.g. adding a role) must be mirrored — small intra-file drift risk, no current drift. +- [SMELL] The single dispatch job is named `resolve` but handles all 8 modes; users see "resolve" in the Actions UI for a /agent-design run. Cosmetic. +- clean otherwise (gate logic, redirect comment, secret plumbing all coherent; RDB_APP_ID via vars context works across workflow_call). + +## .github/workflows/dogfood.yml + +- [BUG] Usage comment (line 8) documents bare `/dogfood` as "resolve with default model", but the job `if` only matches `startsWith('/dogfood-')` or `startsWith('/dogfood ')` — a comment that is exactly `/dogfood` (no dash, no trailing space) never fires. Either the doc line is wrong or the condition needs an exact-match alternative. (agent.yml does not document bare `/agent`, so this is dogfood-only.) +- [SMELL] `permissions: workflows: write` present here but not in agent.yml — presumably intentional (rdb self-dev edits its own workflow files; user repos shouldn't), but undocumented; one comment line would prevent future "why the difference" churn. +- clean otherwise (app_token_owner broadening is well-commented; secrets list matches agent.yml). + +## .github/workflows/test.yml + +- [BUG] **Unit tests never run on PRs to dev.** Trigger is `on: pull_request: branches: [main]`, but the repo's branch model is dev-only — every working PR targets dev. Verified: recently merged dev PRs #631/#632/#633 each show an empty statusCheckRollup (zero CI). Tests only run on the rare dev->main release PR, so breakage lands on dev silently and is discovered in bulk at release time. Fix: `branches: [main, dev]` (pytest run is <5 min, cheap). This is the highest-leverage one-line fix in the whole workflow set. +- [SMELL] `pip install pytest PyYAML litellm` — dependency list duplicated across test.yml / full-test-suite.yml and the big workflow's pip installs with no single requirements file; drift risk (checked below in full-test-suite). + +## .github/workflows/full-test-suite.yml + +- [STALE] Header comment (lines 16-17) claims "e2e jobs use `if: always() && needs.unit-tests.result == 'success'`" — only the `e2e-security` job does; the `e2e` job has a plain `needs: unit-tests` with no `if:`. Functionally equivalent today (e2e is the first e2e block, nothing to continue past), but the comment misdescribes the file and will mislead whoever adds a third e2e block. +- clean otherwise (PyYAML installed unconditionally here, correct since it always runs --all-models; token fallback chain matches e2e.yml/e2e-security.yml). + +## .github/workflows/e2e.yml + +- [SMELL] PyYAML install is gated on `if: inputs.all_models`, but the yaml parse in e2e.sh (line ~108) is reached for any --all-models run including `--test <alias>` selection; if a future refactor moves alias resolution outside the all-models branch, this conditional install becomes a trap. Today it is correct. +- [SMELL] `./tests/e2e.sh $ARGS` with unquoted word-split ARGS built from workflow_dispatch inputs — inputs are maintainer-supplied so not a security issue, but a `test` value with a space silently becomes two args. Low priority. +- clean otherwise. + +## .github/workflows/e2e-security.yml + +- clean — mirrors e2e.yml structure; no PyYAML needed (e2e-security.sh uses only python3 stdlib for rate-limit reset time); UNAUTHORIZED_PAT plumbing matches full-test-suite's e2e-security job. + +## .github/workflows/release.yml + +- [DRIFT] `actions/checkout@v4` while every other workflow in the repo uses `@v5`. Harmless but the only v4 left. +- [SMELL] Changelog extraction: if the tag has no matching `## v<version>` section (or format drifts — regex requires a non-digit char right after the version, which current "## v0.9.0 — ..." headers satisfy), `body` is silently empty and the release is created with no notes. A `[[ -n "$body" ]]` guard with a hard fail would catch tag/changelog mismatches. +- clean otherwise. + +## remote-dev-bot.yml:header+parse + +- [STALE] Header comment (lines 6-10) lists only 4 of 8 modes — workshop, build, and delegate are missing from the "Supports modes" block. +- [STALE] Header line 19: "Supported arguments: max_iterations, timeout_minutes, branch, extra_files" — ALLOWED_ARGS in lib/config.py now has 17 entries (status_log_interval, bash_output_limit, context_keep_tool_results, max_context_tokens, compaction_*, debug_logging, distill_enabled, council, design_rounds, max_design_iterations, ...). Severely understates the surface. +- [BUG-minor] `assign_issue` is declared as a parse job *output* (line 68) but no downstream job ever reads `needs.parse.outputs.assign_issue` (grep count: 0) — it is consumed only inside the parse job itself as `steps.parse.outputs.assign_issue`. Dead output declaration; harmless but misleading. +- clean otherwise (app-token -> checkout -> sparse rdb checkout -> setup-python -> PyYAML -> parse flow is the canonical setup block; rocket reaction and assignment steps are continue-on-error as intended). + +## remote-dev-bot.yml:resolve + +- [BUG] **Safety-net draft-PR fallback can never fire (line 354).** `ISSUE_TYPE="${{ needs.parse.outputs.issue_type }}"` — `issue_type` is not a parse-job output (not in lines 59-89) and lib/config.py never emits it, so the expression is always empty and `[[ "$ON_FAILURE" == "draft" && "$ISSUE_TYPE" == "issue" ]]` is always false. The "resolve.py crashed hard (OOM/SIGKILL) before _cleanup()" draft-PR rescue is dead code. Fix: compute from event context like line 303 does. Same bug at line 906 (build job). +- [BUG] **MAX_ITERATIONS not plumbed into the cleanup steps that read it.** It is set only in the "Resolve issue" step env (line 286). But: "Post result comment" line 396 reads `${MAX_ITERATIONS:-}` (so ITERATIONS_FMT never shows "N / max"), "Append cost to PR" heredoc line 621 reads `os.environ.get("MAX_ITERATIONS")` (same), and "Post cost comment (fallback)" line 709 passes `--max-iterations "${MAX_ITERATIONS:-}"` (always empty). The "iterations / budget" display in PR cost tables and fallback comments silently degrades to a bare count. The job has no job-level env block; fix is either job-level `MAX_ITERATIONS:` or per-step env. Check whether build/delegate/reconcile share this (verified below per job). +- [DRIFT-risk] "Post result comment" re-implements `_fmt_tok` (inline `fmt()` at lines 404-411) and `_fmt_ela` (line 386) instead of importing lib/formatting.py like the "Append cost to PR" heredoc does. Logic is currently byte-identical to formatting.py — no live drift — but formatting.py's docstring claims to be "the single source of truth ... used across workflow Python heredocs", which these inline copies contradict; next change to _fmt_tok will drift. +- [SMELL] Line 634: when /tmp/agent_start_sha is missing, the diff-base fallback is hardcoded `git merge-base HEAD origin/main` — wrong for repos whose default branch is not main and for non-main TARGET_BRANCH runs; produces a bogus Diff/LOC row in the cost table in the rare fallback case. +- [SMELL] The cost-table heredoc inserts `.remote-dev-bot/lib` into sys.path twice (lines 611 and 662) — harmless leftover. + +## remote-dev-bot.yml:build + +- [BUG] Same dead safety-net draft-PR fallback as resolve: line 906 `needs.parse.outputs.issue_type` is always empty. +- [BUG] Same MAX_ITERATIONS-not-plumbed problem as resolve: lines 948, 1295, 1383 read MAX_ITERATIONS in cleanup steps whose env blocks never set it. +- [BUG] **Line 1173: `council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}"` — GitHub expression interpolated directly into a Python string literal inside the council heredoc.** extra_instructions is configured as a YAML block scalar (multi-line) in remote-dev-bot.yaml's documented examples; any newline or double-quote in it produces a Python SyntaxError and the whole council step crashes (or, adversarially, code injection from a target repo's config). The resolve step passes the same value safely via the `env:` block (EXTRA_INSTRUCTIONS); this heredoc should read os.environ instead. Same pattern at lines 2018 (review), 3374-3375 (workshop), 3675-3676 and 4006 (delegate) — 5 council/synthesis steps total, several also interpolating model_extra_instructions. +- [DRIFT] Build's "Write step summary" emits the heading "## Resolve Run Summary" (line 1234) — copy-paste from resolve; should say Build. Cosmetic but visible in the Actions summary UI. +- [DRIFT] Job-level `timeout-minutes` inconsistency across the 8 jobs: parse=5, review=15, design=15, workshop=30, delegate=120, but resolve/build/reconcile have NONE (fall back to the 6h Actions default; the internal awk watchdog only bounds the agent step, not hung cleanup/council steps). +- Setup block (app token / checkout x2 / setup-python / apikey / pip install) verified byte-equivalent to resolve's — no drift. + +## remote-dev-bot.yml:review + +- [BUG] **`review_max_iterations` is consumed but never declared as a parse job output.** lib/config.py writes `review_max_iterations` to GITHUB_OUTPUT (config.py lines 722-723), but the parse job's `outputs:` block (workflow lines 58-89) does not re-export it, so `${{ needs.parse.outputs.review_max_iterations }}` (line 1583) is always empty and the review loop always falls back to the hardcoded default 10. Today that coincidentally equals modes.review.max_iterations: 10, so nothing visibly breaks — but any user override (`max_iterations = 20` on an /agent-review comment, or per-repo config) is silently ignored. Same class of bug as design (below), where the configured value actually differs from the fallback. +- [BUG] Same family at lines 2087 and 2171: REVIEW_MAX_ITERATIONS read in "Post review comment" cost heredoc and the fallback step, but never set in those steps' env (and the parse output is missing anyway) — "N / max" iteration display can never render in review. +- [BUG] Lines 1599-1605 (single-reviewer heredoc): `model = "${{ ... }}"` and `extra_instructions` parts interpolated as Python string literals — same multi-line/quote SyntaxError-injection hazard as build's line 1173 (extra_instructions block scalars are the documented config style). Also line 2018 in the council path. +- [DRIFT] The review heredoc re-implements validate_path / execute_read_file / execute_grep inline (lines 1777-1832) even though lib/tools.py (extracted in a30100c for resolve/reconcile/design_loop) provides the same functions and the heredoc already has `.remote-dev-bot/lib` on sys.path. Limits are currently in sync (50000-char file cap, 100-line grep cap) but the next change to lib/tools.py will not reach review. The review loop itself (~370 lines of heredoc Python) is the largest remaining un-extracted lib-duplicate in the workflow. +- [DRIFT] Setup-block ordering differs from resolve/build: review runs "Install dependencies" + "Generate repo index" *before* "Determine API key"; resolve/build do apikey first, repo index last. Also review's target checkout adds `fetch-depth: '0'` (string, vs unquoted 0 elsewhere) which resolve/build don't pass. Functionally benign; it is exactly the kind of inter-job drift the accepted duplication invites. +- [SMELL] Line 1974: council heredoc uses an *unquoted* `<< PYEOF` delimiter (so bash expands `${PR_NUMBER}` inside the Python source) while every other heredoc in the file uses quoted `<< 'PYEOF'` + env vars. Works, but a `$` anywhere in that Python block becomes a shell expansion — fragile and inconsistent. + +## remote-dev-bot.yml:reconcile + +- [BUG] **Reconcile ignores the context/truncation config — env vars consumed by lib/reconcile.py but never plumbed.** reconcile.py reads BASH_OUTPUT_LIMIT (py-default **8000**), CONTEXT_KEEP_TOOL_RESULTS (py-default **10**), MAX_CONTEXT_TOKENS, COMPACTION_COVERAGE, COMPACTION_FACTOR (lib/reconcile.py:52-56), but the "Reconcile PR" step env block (lines 2289-2307) sets none of them. Result: reconcile runs with bash truncation at 8000 chars and tool-result dropping keep_n=10 — both of which remote-dev-bot.yaml deliberately set to 0, and the keep_n=10 dropping actively defeats the moving-tail prompt cache that was restored in 3831a8f. Note resolve.py hardened the same vars with `_require_env()` (resolve.py:70-71) precisely so missing plumbing crashes loudly; reconcile.py kept silent defaults, so this drifted unnoticed. Fix: add the five env lines to the reconcile step (and consider `_require_env` in reconcile.py). +- [BUG] **"Post cost comment" step: MODEL / ALIAS / MAX_ITER are shell-assigned but never exported (lines 2416-2418), yet the Python heredoc reads them via os.environ (lines 2433, 2457-2458).** Only COST_TARGET_NUMBER gets `export`. Every reconcile cost comment therefore renders the model header as empty backticks and the Iterations row without the "/ max" budget. Trivial fix: `export MODEL ALIAS MAX_ITER`. +- [DRIFT] Reconcile invokes its agent as `python3 .remote-dev-bot/lib/reconcile.py` *without* `-u` (line 2322); resolve and build use `python3 -u` — reconcile's iteration logs buffer instead of streaming, which matters when diagnosing a hung/timed-out run. +- [DRIFT] Reconcile passes `EXTRA_FILES: ${{ needs.parse.outputs.extra_files }}` raw (line 2305) — no "Generate repo index" step, so no /tmp/rdb-index.md is appended (resolve/build/review/design all generate + append it). Plausibly intentional (rebase task needs no index) but undocumented; one comment line would disambiguate drift from design. +- [DRIFT] Reconcile (like resolve/build) lacks DEBUG_LOGGING/DISTILL_ENABLED plumbing — fine for DISTILL (reconcile has no distillation pre-pass) but DEBUG_LOGGING is consumed by shared lib code paths; debug_logging=true config has no effect on reconcile runs. +- [SMELL] "Post cost comment" runs `if: always()`, so after the "Verify this is a PR" guard fails (someone comments /agent-reconcile on an issue), the step still runs and attempts `gh pr comment` with a $0.00 zero-model table; it errors out on a non-PR but adds red noise to the run. Gate on the verify step's outcome. + +## remote-dev-bot.yml:design + +- [BUG] **Distillation is silently broken in design mode — tuple-arity drift between the heredoc and lib/distill.py.** Workflow line 2753 unpacks `maybe_distill()` into **5** variables, but lib/distill.py:477-489 returns a **6-tuple** (…, structural_extract, codebase_total_tokens — the 6th element added when the distillation metric was fixed in 43aae99). The unpack raises ValueError, the broad `except` catches it, prints "Distillation failed … proceeding with full repo context", and every /agent-design run pays full-repo context despite `distill_enabled: true`. lib/design_loop.py:365 was updated to the 6-tuple; the heredoc wasn't. This is the textbook heredoc-vs-lib drift case. +- [BUG] **`design_max_iterations` not declared as a parse job output** (config.py emits it, workflow outputs block lines 58-89 omit it), so `${{ needs.parse.outputs.design_max_iterations }}` (line 2678) is empty and the design loop always runs the hardcoded default **10** instead of the configured **15** (modes.design.max_iterations). Compounding: DESIGN_WRAPUP_ITERATION *is* plumbed and = int(15*0.8) = **12**, computed against the configured budget — so graceful wrapup can never fire in a 10-iteration loop. Design runs therefore end via "exhausted iterations without submit_analysis" more often than designed. Simplest fix: declare the output (or switch the env line to `needs.parse.outputs.max_iterations`, which is already mode-resolved). +- [DRIFT] **The design job does not use lib/design_loop.py at all — yet design_loop.py's module docstring says it was "extracted from the inline Python in the design workflow job so it can be reused by both /agent-design and /agent-workshop".** The extraction left the original ~450-line inline copy in place, and it has already drifted behind the lib version: (1) no `gh` tool (added to design_loop.py in PR #598 — workshop/delegate design stages can inspect issues/PRs/run logs, /agent-design cannot); (2) no iteration-budget paragraph (#603 ba74dce, "all three loops" — actually only lib loops); (3) raw `completion()` instead of `completion_with_retries`; (4) no cache-token tracking and no moving-tail cache markers — /agent-design pays 0% cache hit on Anthropic while workshop's identical Stage-1 loop caches. Replacing the heredoc body with a thin `run_design_loop(...)` call would erase all four drifts at once. (Same argument applies to the review job's inline loop, which has no lib equivalent yet but duplicates lib/tools.py.) +- [BUG] Lines 2696-2702: model/extra_instructions interpolated into Python string literals — same SyntaxError/injection hazard family as build/review. +- [BUG-minor] DESIGN_MAX_ITERATIONS also read in "Post analysis comment" cost heredoc (line 3075) and fallback step (line 3164) whose env blocks never set it — "N / max" display dead here even after the parse-output fix unless env is added to these steps too. +- [SMELL] config.py also emits `reconcile_max_iterations` (config.py:552) which nothing in the workflow declares or reads (reconcile uses the generic `max_iterations` output) — dead output, candidate for removal. +- [SMELL] Design's "Resolve checkout ref" step uses `GH_TOKEN: secrets.RDB_PAT_TOKEN || github.token` (line 2510) — runs *before* the app-token step, so it can't use the app token; every other gh call in the job prefers the app token. Works, but means PR-head lookups use a different identity than the rest of the job — minor inconsistency, can matter for private-repo PAT-less app-only installs (lookup falls to github.token, which is fine in-repo). Borderline; noting for completeness. + +## remote-dev-bot.yml:workshop + +- [BUG] Lines 3359, 3374-3375: alias / extra_instructions / model_extra_instructions interpolated as Python string literals inside the quoted heredoc — same multi-line SyntaxError hazard family (this job otherwise reads its inputs via os.environ; these three are the stragglers). +- Notably clean otherwise — workshop is the model the other jobs should converge on: the agent loop is a thin `run_workshop(...)` call into lib (no inline loop to drift), MAX_ITERATIONS comes from `workshop_max_iterations` which IS a declared parse output, the fallback step reads the parse output directly in the expression (no un-plumbed env var), and the cost-embedded sentinel is written only after the heredoc succeeds (bash -e). + +## remote-dev-bot.yml:delegate + +- [BUG] **Graceful wrapup can never fire in delegate's design stages.** WRAPUP_ITERATION (line 3643) is `graceful_wrapup_iteration` = int(delegate max_iterations 50 × 0.8) = **40**, computed by config.py against the *code-stage* budget. run_delegate passes it unchanged into the Stage-1/Stage-3a design loops (workshop.py:1192, 1438), which run with max_design_iterations = **15** — so the "wrap up now" message is never injected and design stages that run long just die at the iteration cap. Fix: scale wrapup per-stage (int(max_design_iterations × threshold)) either in config.py or in run_delegate. +- [BUG] Lines 3675-3676 and 4006: extra_instructions/model_extra_instructions/alias interpolated as Python literals — same hazard family (counted in build's entry). +- [STALE] Line 4273: the success checklist in the result comment says "6. ✅ Code revision plan" — Stage 6 was upgraded from a one-shot revision *plan* to agentic code revision (per the step's own comment block at lines 4060-4067); the user-facing checklist text wasn't updated. +- [DRIFT] Delegate's checkout uses `ref: ${{ needs.parse.outputs.target_branch }}` directly (line 3510) with no "Resolve checkout ref" PR-head logic (design/workshop have it). Consistent with delegate being issue-only in practice, but nothing gates delegate on issue-type — a /agent-delegate on a PR would design against target_branch while Stage 4's resolve does its own PR checkout. Undocumented assumption. +- Otherwise solid: stage cost merging (save-before-overwrite at 3744-3747 and 4145-4149) is careful; MAX_ITERATIONS/MAX_DESIGN_ITERATIONS are real declared parse outputs; fallback step reads parse output via expression; the no-Append-cost-to-PR rationale (lines 4394-4398, #527 double-count) is documented. + +## tests/e2e.sh + +- [BUG] **Reconcile failures cannot fail the suite.** Line 1361: `total_fail=$((fail + timeout_count + RF_FAIL + TIMEOUT_PHASE_FAIL + WRAPUP_PHASE_FAIL))` — RECONCILE_FAIL (incremented in 6 places, lines 982-1006) is never added, and the final summary block doesn't print a Reconcile line either. A red reconcile test exits 0. +- [BUG] **rf-feedback runs but its result is never verified.** RF_FEEDBACK_RESULT and RF_FEEDBACK_NEW_SHA are captured (lines 786-789) — including the new-SHA check that would prove the feedback commit landed — but the results section prints only rf-resolve and rf-review; nothing evaluates, prints, or counts the feedback step (RF_PASS/RF_FAIL untouched). The step costs a real agent run; a silent rf-feedback failure passes the suite. (The cost-collection section *does* sum rf-feedback's cost, so the omission is clearly accidental.) +- [BUG] **The graceful-wrapup test runs twice.** Lines 406-418 create a wrapup issue + trigger `/agent-resolve max_iterations=4` and the main polling loop waits for it; then "Phase 4" (lines ~1112-1135) overwrites wrapup_ts/wrapup_issue_num/WRAPUP_RUN_ID/WRAPUP_RESULT, creates a *second* identical issue, triggers a second paid agent run, and polls again. The first run's result is discarded (only its issue gets cleaned up). Looks like a refactor leftover — one of the two blocks should go; each costs an LLM run per suite execution. Phase 4 also bypasses the post_issue_comment retry helper (uses raw `gh issue comment`). +- [STALE] Lines 165-167: delegate test comment lists "Stages: 1) design ... 5) council code review" — delegate has had 6 stages since agentic Stage 6 landed. +- [SMELL] `head -16 "$0" | tail -12` for --help (line 36) breaks silently the next time someone edits the header comment block — fragile coupling. +- [SMELL] Phase 4 wrapup PASS criteria accept "PR created" or "could not fully resolve comment" — neither proves the wrapup *instruction* was injected (would need a log grep like the compaction test does). Acknowledged trade-off, noting only because the compaction test shows the stronger pattern is already in use. + +## Top findings (Phase 2) + +1. [BUG] **Distillation silently broken in /agent-design**: workflow heredoc unpacks 5 values from `maybe_distill()` which now returns 6 (lib/distill.py:477); ValueError is swallowed by the broad except and every design run proceeds with full repo context. (remote-dev-bot.yml:2753) +2. [BUG] **No CI on dev PRs**: test.yml triggers only on `pull_request: branches: [main]`; repo model is dev-only PRs — #631-633 merged with zero checks. One-line fix: add dev. (.github/workflows/test.yml:5) +3. [BUG] **Reconcile ignores context config**: BASH_OUTPUT_LIMIT / CONTEXT_KEEP_TOOL_RESULTS / MAX_CONTEXT_TOKENS / COMPACTION_* consumed by lib/reconcile.py but never plumbed into the job env — reconcile runs with truncation=8000 and cache-defeating keep_n=10 against config-mandated 0s. (remote-dev-bot.yml:2289-2307) +4. [BUG] **design_max_iterations / review_max_iterations missing from parse job outputs** (config.py emits them; outputs block doesn't re-export): design loops run 10 iterations instead of the configured 15, inline `max_iterations =` overrides are ignored for design/review, and design's wrapup (fires at 12) can never trigger inside a 10-iteration loop. (remote-dev-bot.yml:58-89 vs 1583/2678) +5. [DRIFT] **The design job never adopted lib/design_loop.py** (whose docstring claims it was extracted for exactly this) — the inline copy lacks the gh tool (#598), iteration-budget hint (#603), retry wrapper, and cache markers (0% cache hit on Anthropic for /agent-design while workshop's identical loop caches at 70-80%). Replace heredoc with run_design_loop(). (remote-dev-bot.yml:2685-3048) +6. [BUG] **`extra_instructions` interpolated into Python string literals in 5 council/loop heredocs** (build 1173, review 1599-1605/2018, workshop 3374-3375, delegate 3675-3676/4006): documented multi-line config values produce SyntaxError; also an injection vector from target-repo config. Pass via env like the resolve steps do. +7. [BUG] **e2e.sh result accounting**: reconcile failures excluded from exit code (line 1361); rf-feedback outcome captured but never verified or counted; wrapup test duplicated (two paid runs, first discarded). +8. [STALE→ACT] **remote-dev-bot.yaml "temporarily disabled to gather baseline data" comments are over**: caching landed (3831a8f) and works. Recommend: bash_output_limit→50000 (design-workspace.md's own conclusion), context_keep_tool_results stays 0 but reword as permanent decision (so nobody "restores" 20 and kills the cache), and add a context-overflow safety valve (max_context_tokens≈180000 or forced-drop-near-limit) since nothing currently guards the model's hard limit. +9. [BUG] **Dead safety-net draft-PR fallback** in resolve+build: `needs.parse.outputs.issue_type` doesn't exist, so the OOM/SIGKILL rescue path can never run (lines 354, 906). Related family: MAX_ITERATIONS not plumbed into cleanup steps (resolve/build/review/design), and reconcile's cost step reads MODEL/ALIAS/MAX_ITER via os.environ without exporting them — empty model header on every reconcile cost comment. +10. [BUG] **Delegate design stages never get the wrapup nudge**: graceful_wrapup_iteration=40 (from max_iterations=50) is passed unchanged into 15-iteration design loops. (remote-dev-bot.yml:3643 → workshop.py:1192) diff --git a/comprehensive-review/README.md b/comprehensive-review/README.md new file mode 100644 index 0000000..407243b --- /dev/null +++ b/comprehensive-review/README.md @@ -0,0 +1,22 @@ +# Comprehensive reviews + +Periodic full-repo review reports — code health, doc/workflow drift, real- +world usage analysis, prompt assessment, recommendations. Distinct from +`design-workspace.md` (a working scratchpad) and the per-PR review comments +on GitHub (per-change, not whole-repo). + +Each review lives in a dated subdirectory. Entry point is `SUMMARY.md`; +detailed findings are in `findings-*.md` alongside it. `PROCESS.md` and +`AGENT-PROMPTS.md` document how the review was produced (sequential +subagents with incremental disk writes — see the Fable budget protocol +memory). + +## Index + +- [`2026-06-11/`](2026-06-11/SUMMARY.md) — first comprehensive review. Done + just after PRs #631-633 merged on dev. Headline findings: two injection + vectors (one shell, one Python heredoc), `install.md` 404 on fresh + install, distillation silently broken in `/agent-design`, `test.yml` + doesn't run on dev PRs. Also: the council layer is the current working + defense against premature-victory and closing the council loop is the + recommended highest-leverage next feature. From 3d1600c504f0248895ee82404b26bdd5a15b6583 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 11:20:33 +0000 Subject: [PATCH 14/39] resolve.py: harden create_pr against title injection + three loop bugs Four fixes in lib/resolve.py, all from the 2026-06-11 comprehensive review (see comprehensive-review/2026-06-11/). 1. create_pr now builds the gh command as an argv list (no shell) instead of interpolating the PR title into a shell=True string with only double-quote escaping. The title is LLM-authored from issue text, so shell metacharacters could reach the runner. argv form removes the shell entirely. 2. Reordered the completion() exception handlers so ContextWindowExceededError is caught before BadRequestError. CWE subclasses BadRequestError, so the old order made the context-overflow recovery (emergency trim + graceful wrap-up) unreachable dead code; overflows died with a generic "Bad request". 3. Cache-write token tracking read getattr(prompt_details, "cache_creation_input_tokens"), but litellm's PromptTokensDetailsWrapper field is cache_creation_tokens. Verified against the installed litellm (model_fields has cache_creation_tokens, not the _input_ variant). The old name always read 0, so cache-write tokens never appeared in status logs and cache-savings figures were overstated. 4. The two "no tool calls for 3 consecutive iterations" break paths now write_status() before breaking. The post-loop code only writes a status when the loop ends naturally, so these breaks previously ended the run with no failure explanation at all. 707 tests pass. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- lib/resolve.py | 103 +++++++++++++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 33 deletions(-) diff --git a/lib/resolve.py b/lib/resolve.py index 3298294..12cbe20 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -1095,21 +1095,32 @@ def execute_tool(tool_name, arguments): def create_pr(branch, pr_title, pr_body, draft=False): """Create a pull request and return its URL.""" - draft_flag = "--draft" if draft else "" body_file = "/tmp/rdb_pr_body.txt" model_header = f"🤖 **Model:** `{ALIAS}` (`{LLM_MODEL}`)\n\n" with open(body_file, "w") as f: f.write(model_header + (pr_body or "")) - # Quote the title carefully - safe_title = pr_title.replace('"', '\\"') - cmd = ( - f'gh pr create --repo {GITHUB_REPO} ' - f'--base {TARGET_BRANCH} --head {branch} ' - f'--title "{safe_title}" ' - f'--body-file {body_file} ' - f'{draft_flag}' - ).strip() - output = run(cmd, timeout=60) + # Build the command as an argv list (no shell) so the title — which is + # LLM-authored from untrusted issue text — cannot inject shell syntax. + # A quoted-string interpolation into `shell=True` would leave backticks + # and $(...) live even after escaping double quotes. + argv = [ + "gh", "pr", "create", + "--repo", GITHUB_REPO, + "--base", TARGET_BRANCH, + "--head", branch, + "--title", pr_title, + "--body-file", body_file, + ] + if draft: + argv.append("--draft") + result = subprocess.run( + argv, capture_output=True, text=True, timeout=60, + ) + output = (result.stdout or "") + (result.stderr or "") + if result.returncode != 0: + raise RuntimeError( + f"Command failed (exit {result.returncode}):\n{' '.join(argv)}\n{output}" + ) match = re.search(r"https://github\.com/[^/\s]+/[^/\s]+/pull/\d+", output) if match: return match.group(0) @@ -1520,28 +1531,13 @@ def main(): max_tokens=16384, transient_error_counter=transient_error_counter, ) - except litellm.exceptions.BadRequestError as exc: - err_msg = str(exc) - if "prefill" in err_msg.lower(): - roles = [m.get("role") for m in messages[-5:]] - print(f"Prefill error at iteration {iteration + 1}. " - f"Last 5 roles: {roles}. Error: {exc}") - write_status(False, f"Bad request at iteration {iteration + 1}: {exc}") - break - except litellm.exceptions.RateLimitError as exc: - rate_limit_retries += 1 - if rate_limit_retries <= MAX_RATE_LIMIT_RETRIES: - wait_secs = 60 * rate_limit_retries - print(f"Rate limit hit (retry {rate_limit_retries}/{MAX_RATE_LIMIT_RETRIES}), " - f"waiting {wait_secs}s: {exc}") - time.sleep(wait_secs) - continue - else: - print(f"Rate limit: exhausted {MAX_RATE_LIMIT_RETRIES} retries, treating as failure.") - write_status(False, f"Rate limit error after {MAX_RATE_LIMIT_RETRIES} retries " - f"at iteration {iteration + 1}: {exc}") - break except litellm.exceptions.ContextWindowExceededError as exc: + # MUST precede the BadRequestError handler below: + # ContextWindowExceededError subclasses BadRequestError, so if + # BadRequest is caught first this block becomes dead code and a + # context overflow dies with a generic "Bad request" instead of + # the emergency trim + graceful wrap-up. + # # Input prompt is too long for the model's context window. # Attempt emergency recovery: force-trim and inject a wrap-up message, # then continue so the agent can commit/push before we exit. @@ -1565,6 +1561,27 @@ def main(): ), }) continue + except litellm.exceptions.BadRequestError as exc: + err_msg = str(exc) + if "prefill" in err_msg.lower(): + roles = [m.get("role") for m in messages[-5:]] + print(f"Prefill error at iteration {iteration + 1}. " + f"Last 5 roles: {roles}. Error: {exc}") + write_status(False, f"Bad request at iteration {iteration + 1}: {exc}") + break + except litellm.exceptions.RateLimitError as exc: + rate_limit_retries += 1 + if rate_limit_retries <= MAX_RATE_LIMIT_RETRIES: + wait_secs = 60 * rate_limit_retries + print(f"Rate limit hit (retry {rate_limit_retries}/{MAX_RATE_LIMIT_RETRIES}), " + f"waiting {wait_secs}s: {exc}") + time.sleep(wait_secs) + continue + else: + print(f"Rate limit: exhausted {MAX_RATE_LIMIT_RETRIES} retries, treating as failure.") + write_status(False, f"Rate limit error after {MAX_RATE_LIMIT_RETRIES} retries " + f"at iteration {iteration + 1}: {exc}") + break except litellm.exceptions.APIConnectionError as exc: err_msg = str(exc) if _is_context_overflow_error(exc): @@ -1629,7 +1646,11 @@ def main(): if prompt_details: iter_cache_read_toks = getattr(prompt_details, "cached_tokens", 0) or 0 total_cache_read_tokens += iter_cache_read_toks - total_cache_creation_tokens += getattr(prompt_details, "cache_creation_input_tokens", 0) or 0 + # litellm's PromptTokensDetailsWrapper field is + # cache_creation_tokens (NOT cache_creation_input_tokens); + # the old name always read as 0, so cache-write tokens + # never appeared and cache-savings figures overstated. + total_cache_creation_tokens += getattr(prompt_details, "cache_creation_tokens", 0) or 0 cost = getattr(response, "_hidden_params", {}).get("response_cost", None) if cost: total_cost += cost @@ -1656,6 +1677,14 @@ def main(): no_tool_call_count += 1 if no_tool_call_count >= 3: print("No tool calls for 3 consecutive iterations — breaking") + # Write a status: the post-loop code only writes one when + # the loop ends naturally, so without this the run reports + # no failure explanation at all. + write_status( + False, + f"Agent stalled at iteration {iteration + 1}: 3 consecutive " + "responses returned no tool call (empty choices).", + ) break print(f"No tool calls (attempt {no_tool_call_count}/3) — injecting recovery message") messages.append( @@ -1679,6 +1708,14 @@ def main(): no_tool_call_count += 1 if no_tool_call_count >= 3: print("No tool calls for 3 consecutive iterations — breaking") + # Write a status: the post-loop code only writes one when + # the loop ends naturally, so without this the run reports + # no failure explanation at all. + write_status( + False, + f"Agent stalled at iteration {iteration + 1}: 3 consecutive " + "responses returned no tool call.", + ) break print(f"No tool calls (attempt {no_tool_call_count}/3) — injecting recovery message") messages.append( From 9ae1becebe2f38bdf335e1a08de022a109cdc674 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 13:08:27 +0000 Subject: [PATCH 15/39] workflow: route config strings through step env + fix plumbing gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four fixes in .github/workflows/remote-dev-bot.yml from the 2026-06-11 comprehensive review (T1.2, T2.2, T2.3, T2.4): 1. Route extra_instructions / model_extra_instructions / model / alias through step `env:` blocks instead of interpolating them as Python string literals inside heredocs (8 sites: build council, review loop, review council, design loop, workshop, delegate stages 1-3, delegate stage-5 council). The resolve steps already used this pattern; the heredoc interpolation crashed on multi-line block-scalar values (the documented config format) and let target-repo config inject Python. 2. Fix the design-job maybe_distill() unpack: the heredoc unpacked 5 values but lib/distill.py returns a 6-tuple, so the swallowed ValueError silently disabled distillation for every /agent-design run. Now matches lib/design_loop.py's 6-value unpack. 3. Declare design_max_iterations and review_max_iterations as parse-job outputs. config.py emits both to GITHUB_OUTPUT but the outputs block never re-exported them, so design/review loops always ran the hardcoded default 10 and per-invocation max_iterations overrides were silently ignored (design's configured budget is 15, and its wrapup fires at 12 — unreachable in a 10-iteration loop). 4. Plumb CONTEXT_KEEP_TOOL_RESULTS / BASH_OUTPUT_LIMIT / MAX_CONTEXT_TOKENS / COMPACTION_COVERAGE / COMPACTION_FACTOR into the reconcile step env. lib/reconcile.py consumes all five but the job never set them, so reconcile ran with truncation=8000 and cache-defeating keep_n=10 against the config-mandated 0s. Behavior changes: design runs get distillation and their configured iteration budget back; reconcile gets its configured context settings; multi-line extra_instructions no longer crashes council/review/design/ workshop/delegate steps. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .github/workflows/remote-dev-bot.yml | 61 +++++++++++++++++++--------- 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/.github/workflows/remote-dev-bot.yml b/.github/workflows/remote-dev-bot.yml index 3e9b8c2..2f9e3cb 100644 --- a/.github/workflows/remote-dev-bot.yml +++ b/.github/workflows/remote-dev-bot.yml @@ -78,6 +78,8 @@ jobs: compaction_factor: ${{ steps.parse.outputs.compaction_factor }} design_context_keep_tool_results: ${{ steps.parse.outputs.design_context_keep_tool_results }} review_context_keep_tool_results: ${{ steps.parse.outputs.review_context_keep_tool_results }} + design_max_iterations: ${{ steps.parse.outputs.design_max_iterations }} + review_max_iterations: ${{ steps.parse.outputs.review_max_iterations }} workshop_max_iterations: ${{ steps.parse.outputs.workshop_max_iterations }} delegate_max_iterations: ${{ steps.parse.outputs.delegate_max_iterations }} delegate_max_design_iterations: ${{ steps.parse.outputs.delegate_max_design_iterations }} @@ -1095,6 +1097,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} COUNCIL_MODELS: ${{ needs.parse.outputs.council_models }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} GITHUB_REPO: ${{ github.repository }} ISSUE_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -1170,7 +1173,7 @@ jobs: council_models = json.loads(os.environ.get("COUNCIL_MODELS", "[]") or "[]") # Mode-level extra_instructions for council reviewers (each model adds its own model-level) - council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}" + council_extra_instructions = os.environ.get("EXTRA_INSTRUCTIONS", "") def post_pr_comment(body): proc = subprocess.run( @@ -1584,6 +1587,9 @@ jobs: REVIEW_CONTEXT_KEEP_TOOL_RESULTS: ${{ needs.parse.outputs.review_context_keep_tool_results }} REVIEW_WRAPUP_ENABLED: ${{ needs.parse.outputs.graceful_wrapup_enabled }} REVIEW_WRAPUP_ITERATION: ${{ needs.parse.outputs.graceful_wrapup_iteration }} + LLM_MODEL: ${{ needs.parse.outputs.model }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} + MODEL_EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.model_extra_instructions }} run: | date +%s > /tmp/start_time python3 << 'PYEOF' @@ -1596,12 +1602,12 @@ jobs: from context import trim_tool_results from litellm import completion - model = "${{ needs.parse.outputs.model }}" + model = os.environ["LLM_MODEL"] - # Load extra_instructions from parse outputs (mode-level + model-level combined) + # Load extra_instructions from the step env (mode-level + model-level combined) extra_instructions = "\n\n".join(p for p in [ - "${{ needs.parse.outputs.extra_instructions }}", - "${{ needs.parse.outputs.model_extra_instructions }}", + os.environ.get("EXTRA_INSTRUCTIONS", ""), + os.environ.get("MODEL_EXTRA_INSTRUCTIONS", ""), ] if p) # Get max iterations from config (default 10) @@ -1966,6 +1972,7 @@ jobs: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} COUNCIL_MODELS: ${{ needs.parse.outputs.council_models }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} GITHUB_REPO: ${{ github.repository }} run: | @@ -2015,7 +2022,7 @@ jobs: issue_body_text = rest.strip() council_models = json.loads(os.environ.get("COUNCIL_MODELS", "[]") or "[]") - council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}" + council_extra_instructions = os.environ.get("EXTRA_INSTRUCTIONS", "") env = {**os.environ, "GH_TOKEN": github_token} @@ -2305,6 +2312,11 @@ jobs: EXTRA_FILES: ${{ needs.parse.outputs.extra_files }} EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} MODEL_EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.model_extra_instructions }} + CONTEXT_KEEP_TOOL_RESULTS: ${{ needs.parse.outputs.context_keep_tool_results }} + BASH_OUTPUT_LIMIT: ${{ needs.parse.outputs.bash_output_limit }} + MAX_CONTEXT_TOKENS: ${{ needs.parse.outputs.max_context_tokens }} + COMPACTION_COVERAGE: ${{ needs.parse.outputs.compaction_coverage }} + COMPACTION_FACTOR: ${{ needs.parse.outputs.compaction_factor }} run: | PR_NUMBER="${{ github.event.issue.number || github.event.pull_request.number }}" @@ -2680,6 +2692,9 @@ jobs: DESIGN_WRAPUP_ENABLED: ${{ needs.parse.outputs.graceful_wrapup_enabled }} DESIGN_WRAPUP_ITERATION: ${{ needs.parse.outputs.graceful_wrapup_iteration }} DISTILL_ENABLED: ${{ needs.parse.outputs.distill_enabled }} + LLM_MODEL: ${{ needs.parse.outputs.model }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} + MODEL_EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.model_extra_instructions }} run: | date +%s > /tmp/start_time python3 << 'PYEOF' @@ -2693,12 +2708,12 @@ jobs: from context import trim_tool_results from litellm import completion - model = "${{ needs.parse.outputs.model }}" + model = os.environ["LLM_MODEL"] - # Load extra_instructions from parse outputs (mode-level + model-level combined) + # Load extra_instructions from the step env (mode-level + model-level combined) extra_instructions = "\n\n".join(p for p in [ - "${{ needs.parse.outputs.extra_instructions }}", - "${{ needs.parse.outputs.model_extra_instructions }}", + os.environ.get("EXTRA_INSTRUCTIONS", ""), + os.environ.get("MODEL_EXTRA_INSTRUCTIONS", ""), ] if p) # Get max iterations from config (default 10) @@ -2750,7 +2765,8 @@ jobs: + (f"\n\n## Discussion so far:\n{comments}" if comments else "") ) print("Running context distillation pre-step (design)...") - distilled, distill_input_tokens, distill_output_tokens, distill_cost, structural_extract = maybe_distill( + (distilled, distill_input_tokens, distill_output_tokens, + distill_cost, structural_extract, _codebase_total) = maybe_distill( repo_context, issue_context_text, model ) if distilled != repo_context: @@ -3338,6 +3354,9 @@ jobs: GITHUB_REPO: ${{ github.repository }} GITHUB_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} MAX_ITERATIONS: ${{ needs.parse.outputs.workshop_max_iterations }} + ALIAS: ${{ needs.parse.outputs.alias }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} + MODEL_EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.model_extra_instructions }} COUNCIL_MODELS: ${{ needs.parse.outputs.council_models }} EXTRA_FILES: ${{ env.RDB_EXTRA_FILES }} WRAPUP_ENABLED: ${{ needs.parse.outputs.graceful_wrapup_enabled }} @@ -3356,7 +3375,7 @@ jobs: from workshop import run_workshop model = os.environ["LLM_MODEL"] - alias = "${{ needs.parse.outputs.alias }}" + alias = os.environ["ALIAS"] issue_number = os.environ["ISSUE_NUMBER"] github_repo = os.environ["GITHUB_REPO"] github_token = os.environ["GITHUB_TOKEN"] @@ -3369,10 +3388,10 @@ jobs: # Council models from parse job (JSON list of {alias, id} dicts) council_models = json.loads(os.environ.get("COUNCIL_MODELS", "[]") or "[]") - # Load extra_instructions from parse outputs + # Load extra_instructions from the step env # council_extra_instructions is mode-level only (each reviewer adds their own model-level) - council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}" - model_extra_instructions = "${{ needs.parse.outputs.model_extra_instructions }}" + council_extra_instructions = os.environ.get("EXTRA_INSTRUCTIONS", "") + model_extra_instructions = os.environ.get("MODEL_EXTRA_INSTRUCTIONS", "") # Stage 1 design agent gets mode + model combined extra_instructions = "\n\n".join(p for p in [council_extra_instructions, model_extra_instructions] if p) @@ -3637,6 +3656,9 @@ jobs: GITHUB_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} MAX_ITERATIONS: ${{ needs.parse.outputs.delegate_max_iterations }} MAX_DESIGN_ITERATIONS: ${{ needs.parse.outputs.delegate_max_design_iterations }} + ALIAS: ${{ needs.parse.outputs.alias }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} + MODEL_EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.model_extra_instructions }} COUNCIL_MODELS: ${{ needs.parse.outputs.council_models }} EXTRA_FILES: ${{ env.RDB_EXTRA_FILES }} WRAPUP_ENABLED: ${{ needs.parse.outputs.graceful_wrapup_enabled }} @@ -3656,7 +3678,7 @@ jobs: from workshop import run_delegate model = os.environ["LLM_MODEL"] - alias = "${{ needs.parse.outputs.alias }}" + alias = os.environ["ALIAS"] issue_number = os.environ["ISSUE_NUMBER"] github_repo = os.environ["GITHUB_REPO"] github_token = os.environ["GITHUB_TOKEN"] @@ -3672,8 +3694,8 @@ jobs: council_models = json.loads(os.environ.get("COUNCIL_MODELS", "[]") or "[]") - council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}" - model_extra_instructions = "${{ needs.parse.outputs.model_extra_instructions }}" + council_extra_instructions = os.environ.get("EXTRA_INSTRUCTIONS", "") + model_extra_instructions = os.environ.get("MODEL_EXTRA_INSTRUCTIONS", "") extra_instructions = "\n\n".join(p for p in [council_extra_instructions, model_extra_instructions] if p) # Generate repository file listing @@ -3926,6 +3948,7 @@ jobs: GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.RDB_PAT_TOKEN || github.token }} COUNCIL_MODELS: ${{ needs.parse.outputs.council_models }} + EXTRA_INSTRUCTIONS: ${{ needs.parse.outputs.extra_instructions }} GITHUB_REPO: ${{ github.repository }} ISSUE_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }} RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -4003,7 +4026,7 @@ jobs: issue_body_text = issue_data.get("body", "") or "" council_models = json.loads(os.environ.get("COUNCIL_MODELS", "[]") or "[]") - council_extra_instructions = "${{ needs.parse.outputs.extra_instructions }}" + council_extra_instructions = os.environ.get("EXTRA_INSTRUCTIONS", "") def post_pr_comment(body): proc = subprocess.run( From e9be96f2c2e1087de124536b67bee5da713e484d Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 13:17:15 +0000 Subject: [PATCH 16/39] test.yml: run CI on PRs targeting dev, not just main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The documented branch model is dev-only PRs — every change lands on dev first and main lags behind. With the trigger limited to main, unit tests never ran on the PRs that constitute actual development (e.g. PRs #631-633 merged with zero CI). From the 2026-06-11 comprehensive review (T1.4). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d3fef0a..92d4303 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Tests on: pull_request: - branches: [main] + branches: [main, dev] jobs: test: From bf80f3e53ed67747a99aeb7a540a7942bb3c325a Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 16:18:21 +0000 Subject: [PATCH 17/39] context.py: keep tool-call pairs intact across the compaction boundary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes from the 2026-06-11 comprehensive review: 1. compact_messages cut the post-system messages at a raw index. When the boundary landed between an assistant(tool_calls) message (which got compacted) and its role="tool" results (which stayed), the new list had orphaned tool results with no preceding tool_use — providers reject that with a 400. The cut now advances to include the whole tool-result group in the compacted set; if that would eat into the protected 2-message recent tail, it retreats so the group is kept intact instead, and no-ops when nothing before the tail is safe to compact. Affects the resolve and reconcile compaction paths. 2. estimate_tokens iterated msg.get("tool_calls", []), which raises TypeError on messages carrying tool_calls: None — a shape raw litellm dicts produce. Now guarded with `or []` like the module's other tool_calls consumers. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/context.py | 24 +++++++++++-- tests/test_compaction.py | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 2 deletions(-) diff --git a/lib/context.py b/lib/context.py index cab1b36..5f5d4c2 100644 --- a/lib/context.py +++ b/lib/context.py @@ -275,8 +275,9 @@ def estimate_tokens(messages): total_chars += len(block.get("text", "")) elif isinstance(block, str): total_chars += len(block) - # Count tool_calls arguments as tokens too - for tc in msg.get("tool_calls", []): + # Count tool_calls arguments as tokens too. Raw litellm message dicts + # can carry tool_calls: None, so guard beyond the missing-key case. + for tc in msg.get("tool_calls") or []: if isinstance(tc, dict): fn = tc.get("function", {}) total_chars += len(fn.get("arguments", "")) @@ -333,6 +334,25 @@ def compact_messages(messages, compaction_coverage, compaction_factor, llm_call_ if n_to_compact >= len(post_system) - 1: n_to_compact = max(1, len(post_system) - 2) + # Never split an assistant(tool_calls) message from its role="tool" + # results: a cut that leaves tool results at the head of the kept + # messages with their tool_use gone produces a list providers reject + # with a 400. Prefer advancing the boundary so the whole group is + # compacted; if that would eat into the protected recent tail, retreat + # so the whole group is kept instead. + cut = n_to_compact + while cut < len(post_system) and post_system[cut].get("role") == "tool": + cut += 1 + if cut > len(post_system) - 2: + cut = n_to_compact + while cut > 0 and post_system[cut].get("role") == "tool": + cut -= 1 + n_to_compact = cut + if n_to_compact == 0: + # The boundary retreated past the start: everything up to the + # protected tail is one tool-call group — nothing safe to compact. + return messages, {"messages_compacted": 0, "tokens_before": 0, "tokens_after": 0} + selected = post_system[:n_to_compact] remaining = post_system[n_to_compact:] diff --git a/tests/test_compaction.py b/tests/test_compaction.py index 16253b8..858a109 100644 --- a/tests/test_compaction.py +++ b/tests/test_compaction.py @@ -36,6 +36,11 @@ def test_tool_calls(self): ] assert estimate_tokens(messages) == (4 + 100) // 4 # "bash" + args + def test_tool_calls_none(self): + """Raw litellm dicts can carry tool_calls: None — must not TypeError.""" + messages = [{"role": "assistant", "content": None, "tool_calls": None}] + assert estimate_tokens(messages) == 0 + def test_multiple_messages(self): messages = [ {"role": "system", "content": "a" * 40}, @@ -157,3 +162,72 @@ def test_handles_tool_calls_in_messages(self): new_msgs, stats = compact_messages(msgs, 0.5, 0.5, self._mock_llm("Summary")) assert stats["messages_compacted"] == 2 assert len(new_msgs) == 4 # system + summary + 2 remaining + + @staticmethod + def _assert_no_orphan_tool_results(msgs): + """A role=tool message must follow an assistant(tool_calls) or another + tool message — otherwise providers reject the list with a 400.""" + for i, msg in enumerate(msgs): + if msg.get("role") == "tool": + prev = msgs[i - 1] if i > 0 else {} + assert prev.get("role") in ("assistant", "tool") and ( + prev.get("role") == "tool" or prev.get("tool_calls") + ), f"orphaned tool result at index {i}" + + def test_boundary_advances_past_tool_results(self): + """A cut landing between assistant(tool_calls) and its tool results + advances so the whole group is compacted, not orphaned.""" + msgs = [ + {"role": "system", "content": "System prompt."}, + {"role": "user", "content": "Please list and read the files. " + "x" * 100}, + {"role": "assistant", "content": None, "tool_calls": [ + {"function": {"name": "bash", "arguments": '{"command": "ls"}'}}, + {"function": {"name": "read_file", "arguments": '{"path": "a.py"}'}}, + ]}, + {"role": "tool", "content": "file1.py\nfile2.py"}, + {"role": "tool", "content": "print('hello')"}, + {"role": "user", "content": "Good, now edit file1.py"}, + {"role": "assistant", "content": "Done editing file1.py."}, + ] + # coverage 1/3 of 6 post-system messages -> raw cut = 2, which would + # orphan the two tool results at indices 3-4. + new_msgs, stats = compact_messages(msgs, 1 / 3, 0.5, self._mock_llm("Summary")) + assert stats["messages_compacted"] == 4 + self._assert_no_orphan_tool_results(new_msgs) + # Message after the summary is the post-group user message + assert new_msgs[2]["role"] == "user" + + def test_boundary_retreats_when_tail_protected(self): + """If advancing past the tool group would eat the protected recent + tail, the cut retreats so the whole group is kept instead.""" + msgs = [ + {"role": "system", "content": "System prompt."}, + {"role": "user", "content": "Please investigate the bug. " + "x" * 100}, + {"role": "assistant", "content": None, "tool_calls": [ + {"function": {"name": "bash", "arguments": '{"command": "pytest"}'}}, + ]}, + {"role": "tool", "content": "1 failed, 42 passed"}, + {"role": "tool", "content": "traceback details"}, + ] + # coverage 0.5 of 4 -> raw cut = 2; advancing to 4 would leave < 2 + # recent messages, so the cut retreats to 1. + new_msgs, stats = compact_messages(msgs, 0.5, 0.5, self._mock_llm("Summary")) + assert stats["messages_compacted"] == 1 + self._assert_no_orphan_tool_results(new_msgs) + # The assistant(tool_calls) + results group survives intact + assert new_msgs[2].get("tool_calls") + + def test_nothing_safe_to_compact_returns_unchanged(self): + """When everything before the protected tail is one tool-call group, + compaction is a no-op rather than producing an invalid list.""" + msgs = [ + {"role": "system", "content": "System prompt."}, + {"role": "assistant", "content": None, "tool_calls": [ + {"function": {"name": "bash", "arguments": '{"command": "ls"}'}}, + ]}, + {"role": "tool", "content": "file1.py"}, + {"role": "tool", "content": "file2.py"}, + ] + new_msgs, stats = compact_messages(msgs, 0.5, 0.5, self._mock_llm("Summary")) + assert stats["messages_compacted"] == 0 + assert new_msgs == msgs From 6799d2bd98b90b3113160823ac9afac3ab5c45c6 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 16:21:08 +0000 Subject: [PATCH 18/39] distill.py: fix signature extraction dropping everything but arg names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _format_function_sig called ast.get_source_segment(source, node.args), which always returns None — ast.arguments nodes carry no position attributes — so every structural-extract signature silently fell back to bare positional argument names: defaults, keyword-only args, *args/**kwargs, and annotations were all dropped. Use ast.unparse(node.args) instead, which reconstructs the full argument spec. From the 2026-06-11 comprehensive review (T2.8). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/distill.py | 22 +++++++++++----------- tests/test_distill.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/lib/distill.py b/lib/distill.py index aa9fd15..2db4e24 100644 --- a/lib/distill.py +++ b/lib/distill.py @@ -250,7 +250,7 @@ def _extract_python_signatures(content, include_line_numbers=False): for node in ast.iter_child_nodes(tree): if isinstance(node, ast.FunctionDef) or isinstance(node, ast.AsyncFunctionDef): # Get the function signature from source lines - sig = _format_function_sig(node, content) + sig = _format_function_sig(node) docstring = ast.get_docstring(node) if include_line_numbers: parts.append(f"{sig} # line {node.lineno}") @@ -271,7 +271,7 @@ def _extract_python_signatures(content, include_line_numbers=False): # Include method signatures for item in ast.iter_child_nodes(node): if isinstance(item, (ast.FunctionDef, ast.AsyncFunctionDef)): - sig = _format_function_sig(item, content) + sig = _format_function_sig(item) if include_line_numbers: parts.append(f" {sig} # line {item.lineno}") else: @@ -288,19 +288,19 @@ def _extract_python_signatures(content, include_line_numbers=False): return "\n".join(parts) if parts else content.split("\n")[:STRUCT_EXTRACT_HEAD_LINES] -def _format_function_sig(node, source): +def _format_function_sig(node): """Format a function/async function node as its signature line.""" prefix = "async def" if isinstance(node, ast.AsyncFunctionDef) else "def" - # Try to get the source segment for just the signature + # ast.arguments nodes carry no position attributes, so + # ast.get_source_segment(source, node.args) always returns None — + # unparse the node instead to keep defaults, *args/**kwargs, + # keyword-only args, and annotations. try: - args = ast.get_source_segment(source, node.args) - if args: - return f"{prefix} {node.name}({args}):" + return f"{prefix} {node.name}({ast.unparse(node.args)}):" except Exception: - pass - # Fallback: reconstruct from AST - arg_names = [a.arg for a in node.args.args] - return f"{prefix} {node.name}({', '.join(arg_names)}):" + # Fallback: reconstruct bare positional names from the AST + arg_names = [a.arg for a in node.args.args] + return f"{prefix} {node.name}({', '.join(arg_names)}):" def format_structural_extract(files, include_line_numbers=False): diff --git a/tests/test_distill.py b/tests/test_distill.py index 709d0f9..8219c9a 100644 --- a/tests/test_distill.py +++ b/tests/test_distill.py @@ -300,6 +300,34 @@ def method(self, x: int): assert "MyClass" in result assert "method" in result + def test_python_signatures_keep_full_argument_spec(self): + """Signatures must keep defaults, *args/**kwargs, keyword-only args, + and annotations — not degrade to bare positional names (the old + ast.get_source_segment(source, node.args) path always returned None + because ast.arguments carries no position attributes).""" + py_content = ''' +def full(a, b: int = 3, *args, key: str = "x", **kwargs) -> bool: + """Docstring.""" + return True + +async def afull(timeout: float = 1.5): + pass + +class C: + def method(self, x: int = 0, *, flag=False): + pass +''' + files = [ + {"path": "mod.py", "content": py_content, "is_source": True, "truncated": False}, + ] + result = format_structural_extract(files) + # ast.unparse spacing around annotated defaults varies across Python + # versions ("int=3" vs "int = 3") — compare with whitespace stripped. + squeezed = result.replace(" ", "") + assert "deffull(a,b:int=3,*args,key:str='x',**kwargs):" in squeezed + assert "asyncdefafull(timeout:float=1.5):" in squeezed + assert "defmethod(self,x:int=0,*,flag=False):" in squeezed + def test_non_python_first_lines(self): js_content = "\n".join([f"line {i}" for i in range(50)]) files = [ From dd51d369fec1142935f48d14db35cb5a92018e95 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 16:23:06 +0000 Subject: [PATCH 19/39] reconcile.py: four loop fixes from the comprehensive review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Wrap the main-loop completion() in completion_with_retries, like resolve and design_loop already do — a single transient provider error (e.g. Anthropic 529 "overloaded") no longer kills the run via the generic APIError handler. 2. Read cache-write tokens from litellm's actual field name cache_creation_tokens (was cache_creation_input_tokens, which always read 0 — cache-write tokens never surfaced and cache-savings figures overstated). Same fix as resolve.py in #636; T2.5. 3. Write a failure status before the "no tool calls x3" break — the post-loop logic only writes a status on natural completion, so this exit previously ended the run with no explanation. Same shape as resolve.py's fix in #636; T2.7. 4. Fix the conflict-marker side explanation in RECONCILE_WORKFLOW: during `git rebase origin/BASE`, the side above ======= (HEAD/ours) is origin/BASE and the side below is the PR branch's commit being replayed. The prompt attributed the below side to origin/BASE — backwards — priming the agent to misattribute intent on every conflict. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/reconcile.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/lib/reconcile.py b/lib/reconcile.py index 7ba5b73..7a77d93 100644 --- a/lib/reconcile.py +++ b/lib/reconcile.py @@ -27,6 +27,7 @@ from lib.context import ( classify_provider_error, compact_messages, + completion_with_retries, estimate_tokens, ) from lib.tools import ( @@ -296,8 +297,12 @@ def write_usage(input_tokens, output_tokens, cost, iterations, 1. **Read the file** — use read_file to see the full conflict markers. 2. **Understand both sides** — before writing anything, explain: - - What the BASE side (above `=======`) was trying to do - - What the INCOMING side (below `=======`, from origin/{BASE_BRANCH}) was trying to do + - What the BASE side (above `=======`, from origin/{BASE_BRANCH}) was trying to do + - What the INCOMING side (below `=======`, the PR branch's commit being replayed) was trying to do + + Note the sides during a rebase are the reverse of a merge: HEAD (above + `=======`) is origin/{BASE_BRANCH}, and the "incoming" side below is + this PR's own commit being replayed on top. 3. **Write the resolved file** — produce a merged result that preserves both intentions. Use `bash` to write the resolved content: `cat > path << 'RESOLVED_EOF' ... RESOLVED_EOF` 4. **Stage the file**: `git add <file>` @@ -543,7 +548,11 @@ def main(): _last_blk["cache_control"] = _cc try: - response = completion( + # Wrapped like the sibling loops (resolve, design_loop) so a + # transient provider error (e.g. Anthropic 529 "overloaded") + # is retried instead of killing the run. + response = completion_with_retries( + completion, model=LLM_MODEL, messages=messages, tools=TOOLS, @@ -595,7 +604,11 @@ def main(): prompt_details = getattr(usage, "prompt_tokens_details", None) if prompt_details: total_cache_read_tokens += getattr(prompt_details, "cached_tokens", 0) or 0 - total_cache_creation_tokens += getattr(prompt_details, "cache_creation_input_tokens", 0) or 0 + # litellm's PromptTokensDetailsWrapper field is + # cache_creation_tokens (NOT cache_creation_input_tokens); + # the old name always read as 0, so cache-write tokens + # never appeared and cache-savings figures overstated. + total_cache_creation_tokens += getattr(prompt_details, "cache_creation_tokens", 0) or 0 cost = getattr(response, "_hidden_params", {}).get("response_cost", None) if cost: total_cost += cost @@ -608,6 +621,14 @@ def main(): no_tool_call_count += 1 if no_tool_call_count >= 3: print("No tool calls for 3 consecutive iterations — breaking") + # Write a status: the post-loop code only writes one when + # the loop ends naturally, so without this the run reports + # no failure explanation at all. + write_status( + False, + f"Agent stalled at iteration {iteration + 1}: 3 consecutive " + "responses returned no tool call.", + ) break print(f"No tool calls (attempt {no_tool_call_count}/3) — injecting recovery message") messages.append({ From 33eeb81dc07e25b90e711b74d77ed333fe9fe12e Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 16:24:06 +0000 Subject: [PATCH 20/39] design_loop.py: read cache-write tokens from litellm's actual field name MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cache_creation_input_tokens does not exist on litellm's PromptTokensDetailsWrapper — the field is cache_creation_tokens — so the getattr always returned 0: cache-write tokens never surfaced in status logs and cache-savings figures overstated (write overhead was never subtracted). Same fix as resolve.py (#636) and reconcile.py; from the 2026-06-11 comprehensive review (T2.5). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/design_loop.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/design_loop.py b/lib/design_loop.py index 1d5e663..524d083 100644 --- a/lib/design_loop.py +++ b/lib/design_loop.py @@ -418,7 +418,11 @@ def run_design_loop( prompt_details = getattr(usage, "prompt_tokens_details", None) if prompt_details: total_cache_read_tokens += getattr(prompt_details, "cached_tokens", 0) or 0 - total_cache_creation_tokens += getattr(prompt_details, "cache_creation_input_tokens", 0) or 0 + # litellm's PromptTokensDetailsWrapper field is + # cache_creation_tokens (NOT cache_creation_input_tokens); + # the old name always read as 0, so cache-write tokens + # never appeared and cache-savings figures overstated. + total_cache_creation_tokens += getattr(prompt_details, "cache_creation_tokens", 0) or 0 cost = getattr(response, "_hidden_params", {}).get("response_cost", None) if cost: total_cost += cost From 227c13576f07213a255558c58e93c194c30eeb88 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 17:23:20 +0000 Subject: [PATCH 21/39] Auto-detect default_model from configured API keys; fix install docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fresh install previously failed on run 1 unless the user created a per-repo config pinning default_model to match their API key — and the install.md step for that pointed at remote-dev-bot.yaml.template, a file accidentally deleted in 820f891, so the documented curl 404s. Two-part fix so the file is truly optional: Behavior: the base config now ships `default_model: auto`, resolved in lib/config.py from whichever provider API-key env vars are present, in priority order ANTHROPIC_API_KEY (claude-small) > OPENAI_API_KEY (gpt-small) > GEMINI_API_KEY (gemini-small) — anthropic first preserves the historical default for multi-key installs. Anything explicitly configured (target-repo config, mode-level default, model suffix on the command) bypasses detection; truly-unset behaves like auto. No key at all fails at parse time with an actionable message instead of a provider auth error mid-run. The parse step gets the three key secrets in env (presence-probe only). The base config's redundant per-mode `default_model: claude-small` entries are removed: under deep_merge they shadowed both the global sentinel and any target repo's global default_model override. Docs: install.md Step 2.3.1 no longer curls the deleted template; it now says the per-repo config is optional, documents the detection priority, and points at the rdb repo's own remote-dev-bot.yaml as the live reference for available options. Tests: pin provider-key env state in test_config.py so resolution is deterministic under any developer environment, plus 10 new tests covering single-key detection per provider, multi-key tie-breaks, empty-string keys, explicit-override-wins (config and command), the unset-config fallback, and the no-keys error. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .github/workflows/remote-dev-bot.yml | 5 ++ install.md | 39 +++++---- lib/config.py | 38 ++++++++- remote-dev-bot.yaml | 16 ++-- tests/test_config.py | 119 +++++++++++++++++++++++++++ tests/test_yaml.py | 3 +- 6 files changed, 191 insertions(+), 29 deletions(-) diff --git a/.github/workflows/remote-dev-bot.yml b/.github/workflows/remote-dev-bot.yml index 2f9e3cb..e6da516 100644 --- a/.github/workflows/remote-dev-bot.yml +++ b/.github/workflows/remote-dev-bot.yml @@ -130,6 +130,11 @@ jobs: env: COMMENT_BODY: ${{ github.event.comment.body }} COMMAND_PREFIX: ${{ inputs.command_prefix || 'agent' }} + # Provider keys are only probed for presence: `default_model: auto` + # resolves to the first provider with a configured key. + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} run: | # Pass full comment body to config.py via COMMENT_BODY env var. # COMMAND_PREFIX tells config.py which slash command prefix to expect. diff --git a/install.md b/install.md index ccf4ec6..76ccdbd 100644 --- a/install.md +++ b/install.md @@ -447,32 +447,31 @@ gh secret list --repo {owner}/{repo} # Should list the secrets you just set (values are hidden) ``` -### Step 2.3.1: Create remote-dev-bot.yaml +### Step 2.3.1: (Optional) Create remote-dev-bot.yaml -**What this does:** Sets `default_model` in your repo's config file to match -the API key you just added. Without this step, the bot defaults to a Claude -model — which will fail on the first run if you added a Gemini or OpenAI key -instead. +A per-repo `remote-dev-bot.yaml` is **optional** — the bot runs on sensible +defaults without one. In particular, you do NOT need to configure a model to +match your API key: the default is `default_model: auto`, which picks the +small model of whichever provider key you added in Step 2.3. If you added +more than one key, the priority order is `ANTHROPIC_API_KEY` (claude-small) > +`OPENAI_API_KEY` (gpt-small) > `GEMINI_API_KEY` (gemini-small). -Start from the provided template, which includes commented examples of the most -useful options: +Create the file only if you want to override a default — pin a specific +model, change iteration limits, add per-mode extra instructions, and so on. +Put a `remote-dev-bot.yaml` in the repo root containing just the keys you +want to change; everything else inherits from the base config. For the live +reference of every available option and its default, see the rdb repo's own +config: -```bash -# From within the target repo: -curl -o remote-dev-bot.yaml \ - https://raw.githubusercontent.com/gnovak/remote-dev-bot/main/remote-dev-bot.yaml.template -``` +https://github.com/gnovak/remote-dev-bot/blob/main/remote-dev-bot.yaml -Then open the file and uncomment the `default_model` line that matches your API -key: +For example, to pin a model instead of auto-detection: -- `default_model: claude-small` — if you added `ANTHROPIC_API_KEY` -- `default_model: gemini-small` — if you added `GEMINI_API_KEY` -- `default_model: gpt-small` — if you added `OPENAI_API_KEY` +```yaml +default_model: gemini-small +``` -The rest of the file shows available options with explanations — all commented -out, so there are no active overrides until you choose to enable them. Browse -it to see what's configurable; you can always come back and tweak later. +You can always come back and add this later. ### Step 2.4: Bot Identity & CI Triggering diff --git a/lib/config.py b/lib/config.py index 2fe866a..cb8640b 100644 --- a/lib/config.py +++ b/lib/config.py @@ -83,6 +83,15 @@ def normalize_config(config): KNOWN_PROVIDERS = ("anthropic/", "openai/", "gemini/") +# `default_model: auto` resolves to the first entry whose API-key env var is +# set. anthropic comes first because claude-small is the historical default, +# so multi-key installs keep their previous behavior. +AUTO_MODEL_PRIORITY = ( + ("ANTHROPIC_API_KEY", "claude-small"), + ("OPENAI_API_KEY", "gpt-small"), + ("GEMINI_API_KEY", "gemini-small"), +) + DEFAULT_TIMEOUT_MINUTES = 120 # Arguments that can be overridden via inline args (lines after the command) @@ -263,6 +272,26 @@ def detect_api_provider(model_id): raise ValueError(f"Unknown provider for model: {model_id}") +def resolve_auto_model(): + """Resolve the `auto` default_model sentinel from present API keys. + + Returns the model alias for the highest-priority provider whose API-key + env var is non-empty (see AUTO_MODEL_PRIORITY). Raises ValueError when + no provider key is available, so a fresh install gets an actionable + message instead of a provider auth failure mid-run. + """ + for env_var, alias in AUTO_MODEL_PRIORITY: + if os.environ.get(env_var, "").strip(): + return alias + raise ValueError( + "default_model is 'auto' but no provider API key is available. " + "Add one of " + + ", ".join(env_var for env_var, _ in AUTO_MODEL_PRIORITY) + + " as a repository secret, or set default_model explicitly in " + "remote-dev-bot.yaml." + ) + + def parse_command(command_string, known_modes): """Parse a command string into (mode, model_alias). @@ -384,7 +413,14 @@ def resolve_config(base_path, override_path, command_string, local_path=None, ti # Resolve model alias — use mode's default if none specified if not alias: - alias = mode_config.get("default_model", config.get("default_model", "claude-small")) + alias = mode_config.get("default_model", config.get("default_model", "auto")) + + # The `auto` sentinel (base-config default) resolves from whichever + # provider API keys are present. Anything explicitly configured — target + # repo config, mode-level default, or a model suffix on the command — + # bypasses this entirely. + if alias == "auto": + alias = resolve_auto_model() models = config.get("models", {}) if alias not in models: diff --git a/remote-dev-bot.yaml b/remote-dev-bot.yaml index a79fc53..cb8995f 100644 --- a/remote-dev-bot.yaml +++ b/remote-dev-bot.yaml @@ -12,7 +12,11 @@ # To use a specific model, append it: /agent-resolve-claude-large # This works for any mode. See models: below for available aliases. -default_model: claude-small +# auto = pick the small model of whichever provider API key is configured, +# checked in priority order: ANTHROPIC_API_KEY (claude-small) > +# OPENAI_API_KEY (gpt-small) > GEMINI_API_KEY (gemini-small). +# Set an explicit alias (e.g. claude-small) to pin a model instead. +default_model: auto # Modes define what the agent does. Each mode specifies optional defaults; # omit a key to inherit from the agent: section below. @@ -24,7 +28,8 @@ default_model: claude-small # need light orientation files; extra context would add noise modes: resolve: - default_model: claude-small + # Optional: pin a mode-specific model (overrides the global default_model). + # default_model: claude-small max_iterations: 50 # Iteration limit for the agent loop extra_files: - AGENTS.md @@ -34,7 +39,6 @@ modes: # Always add type annotations to any Python functions you write. design: - default_model: claude-small max_iterations: 15 # Iteration limit for the agentic loop extra_files: - README.md @@ -61,7 +65,8 @@ modes: - gemini-small workshop: - default_model: claude-small # Model for design stage (Stage 1) + # Optional: pin the design-stage (Stage 1) model. + # default_model: claude-small max_iterations: 15 # Iteration limit for the design loop extra_files: - README.md @@ -79,7 +84,6 @@ modes: - gemini-small build: - default_model: claude-small max_iterations: 50 # Iteration limit for the implementation loop extra_files: - AGENTS.md @@ -91,7 +95,6 @@ modes: - gemini-small reconcile: - default_model: claude-small max_iterations: 50 # Iteration limit for the rebase + conflict resolution loop extra_files: - AGENTS.md @@ -99,7 +102,6 @@ modes: delegate: - default_model: claude-small # Delegate has two independent iteration budgets: # max_iterations — code-writing stages (Stage 4 resolve, Stage 6 # agentic code revision). Same name as elsewhere diff --git a/tests/test_config.py b/tests/test_config.py index d5bd077..9a61853 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -20,6 +20,18 @@ ) +@pytest.fixture(autouse=True) +def pin_provider_keys(monkeypatch): + """Pin provider API-key env state so `default_model: auto` resolution is + deterministic regardless of the developer's real environment: only + ANTHROPIC_API_KEY is set, so auto resolves to claude-small (the + historical default). Tests exercising other combinations override this. + """ + monkeypatch.setenv("ANTHROPIC_API_KEY", "test-key") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.delenv("GEMINI_API_KEY", raising=False) + + # --- deep_merge --- @@ -2441,3 +2453,110 @@ def test_target_branch_explicit_true_via_args(self, base_config_path): ) assert result["target_branch_explicit"] is True assert result["target_branch"] == "my-feature" + + +# --- default_model: auto (API-key detection) --- + + +class TestAutoDefaultModel: + """`default_model: auto` resolves from whichever provider API keys are set. + + Priority: ANTHROPIC_API_KEY > OPENAI_API_KEY > GEMINI_API_KEY (anthropic + first preserves the historical claude-small default for multi-key + installs). Anything explicitly configured bypasses detection. + """ + + KEYS = ("ANTHROPIC_API_KEY", "OPENAI_API_KEY", "GEMINI_API_KEY") + + @pytest.fixture + def auto_config_dir(self, tmp_path, monkeypatch): + """Base config with default_model: auto; all provider keys unset.""" + for key in self.KEYS: + monkeypatch.delenv(key, raising=False) + base = tmp_path / "base" + base.mkdir() + config = { + "default_model": "auto", + "models": { + "claude-small": {"id": "anthropic/claude-sonnet-4-6"}, + "gpt-small": {"id": "openai/gpt-5.1-codex-mini"}, + "gemini-small": {"id": "gemini/gemini-2.5-flash"}, + }, + "modes": {"resolve": {"max_iterations": 50}}, + "agent": {"max_iterations": 50}, + } + (base / "remote-dev-bot.yaml").write_text(yaml.dump(config)) + return tmp_path, str(base / "remote-dev-bot.yaml") + + def test_anthropic_key_only(self, auto_config_dir, monkeypatch): + _, base_path = auto_config_dir + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve") + assert result["alias"] == "claude-small" + assert result["model"] == "anthropic/claude-sonnet-4-6" + + def test_openai_key_only(self, auto_config_dir, monkeypatch): + _, base_path = auto_config_dir + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve") + assert result["alias"] == "gpt-small" + + def test_gemini_key_only(self, auto_config_dir, monkeypatch): + _, base_path = auto_config_dir + monkeypatch.setenv("GEMINI_API_KEY", "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve") + assert result["alias"] == "gemini-small" + + def test_multiple_keys_anthropic_wins(self, auto_config_dir, monkeypatch): + _, base_path = auto_config_dir + for key in self.KEYS: + monkeypatch.setenv(key, "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve") + assert result["alias"] == "claude-small" + + def test_openai_and_gemini_openai_wins(self, auto_config_dir, monkeypatch): + _, base_path = auto_config_dir + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setenv("GEMINI_API_KEY", "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve") + assert result["alias"] == "gpt-small" + + def test_empty_key_does_not_count(self, auto_config_dir, monkeypatch): + """A key set to the empty string is treated as absent.""" + _, base_path = auto_config_dir + monkeypatch.setenv("ANTHROPIC_API_KEY", "") + monkeypatch.setenv("GEMINI_API_KEY", "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve") + assert result["alias"] == "gemini-small" + + def test_no_keys_raises_actionable_error(self, auto_config_dir): + _, base_path = auto_config_dir + with pytest.raises(ValueError, match="no provider API key"): + resolve_config(base_path, "nonexistent.yaml", "resolve") + + def test_explicit_override_config_wins(self, auto_config_dir, monkeypatch): + """A target repo's explicit default_model beats key detection.""" + tmp_path, base_path = auto_config_dir + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + override = tmp_path / "override.yaml" + override.write_text(yaml.dump({"default_model": "gemini-small"})) + result = resolve_config(base_path, str(override), "resolve") + assert result["alias"] == "gemini-small" + + def test_explicit_command_model_wins(self, auto_config_dir, monkeypatch): + """A model suffix on the command beats key detection.""" + _, base_path = auto_config_dir + monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-test") + result = resolve_config(base_path, "nonexistent.yaml", "resolve-gpt-small") + assert result["alias"] == "gpt-small" + + def test_unset_default_model_falls_back_to_auto(self, auto_config_dir, monkeypatch): + """A config with no default_model at all behaves like auto.""" + tmp_path, base_path = auto_config_dir + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + config = yaml.safe_load(open(base_path)) + del config["default_model"] + no_default = tmp_path / "no-default.yaml" + no_default.write_text(yaml.dump(config)) + result = resolve_config(str(no_default), "nonexistent.yaml", "resolve") + assert result["alias"] == "gpt-small" diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 90b473d..166b632 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -52,7 +52,8 @@ def test_config_has_required_keys(bot_config): def test_default_model_exists_in_models(bot_config): default = bot_config["default_model"] - assert default in bot_config["models"], ( + # `auto` is the detect-from-API-keys sentinel, resolved in config.py + assert default == "auto" or default in bot_config["models"], ( f"default_model '{default}' not in models" ) From 93c1936e5b302f3d5143e2f9014561aef1a573cb Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 17:31:45 +0000 Subject: [PATCH 22/39] resolve.py: use lib/context.py's trim_tool_results, drop stale local copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve carried its own trim_tool_results — a stale copy of the pre-smart-ordering implementation that dropped the oldest tool results regardless of kind. The shared version in lib/context.py (already used by reconcile and design_loop) drops read-only bash results first, read_file/grep second, and bash write results (commits, file edits) last. resolve — the one loop that writes code — was ironically the only loop without the protect-write-results behavior, and its local copy was untested (the suite covers the lib version). Behavior change: resolve gains read/write-aware drop ordering. From the 2026-06-11 comprehensive review (T2.6). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/resolve.py | 78 +------------------------------------------------- 1 file changed, 1 insertion(+), 77 deletions(-) diff --git a/lib/resolve.py b/lib/resolve.py index 12cbe20..91e8c6e 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -31,6 +31,7 @@ compact_messages, completion_with_retries, estimate_tokens, + trim_tool_results, ) from lib.tools import ( validate_path, @@ -999,83 +1000,6 @@ def build_system_prompt(repo_context, issue_context_str, distillation_ran=False) -def trim_tool_results(messages, keep_n): - """Remove oldest tool call/result pairs, keeping the last keep_n pairs. - - Preserves all assistant text content and the system prompt. - Operates on OpenAI-format messages where tool calls are in assistant messages - (role="assistant", tool_calls=[...]) and results are role="tool" messages. - """ - if keep_n <= 0: - return messages - - # Collect indices of all "tool" role messages (each is one tool result) - tool_result_indices = [i for i, m in enumerate(messages) if m.get("role") == "tool"] - - if len(tool_result_indices) <= keep_n: - return messages - - # Number of pairs to drop - n_drop = len(tool_result_indices) - keep_n - indices_to_drop = set(tool_result_indices[:n_drop]) - - # Also find the assistant messages that contain tool_calls for the pairs we're dropping. - # Each assistant message with tool_calls is immediately followed by one or more tool messages. - # We scan backwards from each tool_result index to find its owning assistant message. - dropped_tool_call_ids = set() - for idx in indices_to_drop: - dropped_tool_call_ids.add(messages[idx].get("tool_call_id")) - - new_messages = [] - for i, msg in enumerate(messages): - role = msg.get("role") - if i in indices_to_drop: - # Drop this tool result message entirely - continue - if role == "assistant" and msg.get("tool_calls"): - # Filter out tool_calls whose IDs are being dropped - remaining_calls = [ - tc for tc in msg["tool_calls"] - if tc.get("id") not in dropped_tool_call_ids - ] - dropped_calls = [ - tc for tc in msg["tool_calls"] - if tc.get("id") in dropped_tool_call_ids - ] - if dropped_calls: - # Build a new assistant message: preserve content, replace dropped calls - # with a placeholder text note. Keep remaining tool calls if any. - n_omitted = len(dropped_calls) - placeholder_text = f"[{n_omitted} tool call(s) omitted for context]" - new_msg = dict(msg) - if remaining_calls: - new_msg["tool_calls"] = remaining_calls - # Prepend placeholder to content (content may be str or list or None) - if new_msg.get("content") is None: - new_msg["content"] = placeholder_text - elif isinstance(new_msg["content"], str): - new_msg["content"] = placeholder_text + "\n" + new_msg["content"] - else: - # list of content blocks — prepend text block - new_msg["content"] = [{"type": "text", "text": placeholder_text}] + list(new_msg["content"]) - else: - # No remaining calls — strip tool_calls entirely, keep only content - new_msg = {"role": "assistant"} - if msg.get("content") is None or msg.get("content") == []: - new_msg["content"] = placeholder_text - elif isinstance(msg["content"], str): - new_msg["content"] = (msg["content"] + "\n" + placeholder_text).strip() - else: - new_msg["content"] = list(msg["content"]) + [{"type": "text", "text": placeholder_text}] - new_messages.append(new_msg) - else: - new_messages.append(msg) - else: - new_messages.append(msg) - - return new_messages - - def execute_tool(tool_name, arguments): """Dispatch a tool call and return the result string.""" if tool_name == "bash": From 39b11087f23659c491c92770ec427374dceced2c Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 17:38:42 +0000 Subject: [PATCH 23/39] workshop.py: three delegate fixes from the comprehensive review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Stage 3 /agent-command blocking now discards the contaminated revision instead of only suppressing its comment post. Previously the revised_design still flowed into Stages 3a/3b/3c and the workflow's Stage 4, making the "blocked for safety" message cosmetic (Stage 3c already nulled its artifact — the two stages were inconsistent). Falls back to the Stage 1 design, which was /agent-checked upstream. 2. Graceful wrapup can now fire in delegate's design stages. wrapup_iteration is computed by config.py against the code-stage budget (40 of 50) and was passed unchanged into 15-iteration design loops, where it was unreachable. run_delegate now rescales it to the design budget (40/50 -> 12 of 15) for Stage 1 and Stage 3a. 3. Distillation cost included in workshop/delegate totals. run_design_loop reports distill_input_tokens / distill_output_tokens / distill_cost separately from its loop fields; run_workshop and run_delegate summed only the loop fields, under-reporting every Stage 1/3a run's real cost. resolve.py already seeds its totals with distillation; the orchestrators now match. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/workshop.py | 83 ++++++++++++++++++++++++---------- tests/test_workshop.py | 100 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 24 deletions(-) diff --git a/lib/workshop.py b/lib/workshop.py index 3776183..7203adc 100644 --- a/lib/workshop.py +++ b/lib/workshop.py @@ -695,9 +695,12 @@ def post(body): return { "design_result": design_result, "council_results": [], - "total_input_tokens": design_result.get("input_tokens", 0), - "total_output_tokens": design_result.get("output_tokens", 0), - "total_cost": design_result.get("cost", 0.0), + "total_input_tokens": design_result.get("input_tokens", 0) + + design_result.get("distill_input_tokens", 0), + "total_output_tokens": design_result.get("output_tokens", 0) + + design_result.get("distill_output_tokens", 0), + "total_cost": design_result.get("cost", 0.0) + + design_result.get("distill_cost", 0.0), } # Check for agent command loop @@ -709,9 +712,12 @@ def post(body): return { "design_result": design_result, "council_results": [], - "total_input_tokens": design_result.get("input_tokens", 0), - "total_output_tokens": design_result.get("output_tokens", 0), - "total_cost": design_result.get("cost", 0.0), + "total_input_tokens": design_result.get("input_tokens", 0) + + design_result.get("distill_input_tokens", 0), + "total_output_tokens": design_result.get("output_tokens", 0) + + design_result.get("distill_output_tokens", 0), + "total_cost": design_result.get("cost", 0.0) + + design_result.get("distill_cost", 0.0), } # Post design analysis with embedded cost table @@ -741,9 +747,12 @@ def post(body): return { "design_result": design_result, "council_results": [], - "total_input_tokens": design_result.get("input_tokens", 0), - "total_output_tokens": design_result.get("output_tokens", 0), - "total_cost": design_result.get("cost", 0.0), + "total_input_tokens": design_result.get("input_tokens", 0) + + design_result.get("distill_input_tokens", 0), + "total_output_tokens": design_result.get("output_tokens", 0) + + design_result.get("distill_output_tokens", 0), + "total_cost": design_result.get("cost", 0.0) + + design_result.get("distill_cost", 0.0), } post( @@ -842,15 +851,23 @@ def _run_single_review(council_model): f"- Post `/agent-resolve` to implement directly\n" ) - # Aggregate totals - total_input = design_result.get("input_tokens", 0) + sum( - cr.get("input_tokens", 0) for cr in council_results + # Aggregate totals. The distillation pre-pass reports its tokens/cost + # separately from the loop's own fields, so include it here — resolve.py + # seeds its totals the same way. + total_input = ( + design_result.get("input_tokens", 0) + + design_result.get("distill_input_tokens", 0) + + sum(cr.get("input_tokens", 0) for cr in council_results) ) - total_output = design_result.get("output_tokens", 0) + sum( - cr.get("output_tokens", 0) for cr in council_results + total_output = ( + design_result.get("output_tokens", 0) + + design_result.get("distill_output_tokens", 0) + + sum(cr.get("output_tokens", 0) for cr in council_results) ) - total_cost = design_result.get("cost", 0.0) + sum( - cr.get("cost", 0.0) for cr in council_results + total_cost = ( + design_result.get("cost", 0.0) + + design_result.get("distill_cost", 0.0) + + sum(cr.get("cost", 0.0) for cr in council_results) ) return { @@ -1168,6 +1185,17 @@ def post(body): if max_design_iterations is None: max_design_iterations = max_iterations + # wrapup_iteration is computed by config.py against the code-stage budget + # (max_iterations, e.g. 40 of 50). Rescale it for the shorter design + # stages so the wrap-up nudge can actually fire there (e.g. 12 of 15) — + # passed unscaled it exceeds max_design_iterations and is unreachable. + if wrapup_iteration and max_iterations: + design_wrapup_iteration = int( + max_design_iterations * wrapup_iteration / max_iterations + ) + else: + design_wrapup_iteration = wrapup_iteration + all_input_tokens = 0 all_output_tokens = 0 all_cost = 0.0 @@ -1190,16 +1218,18 @@ def post(body): extra_context=extra_context, max_iterations=max_design_iterations, wrapup_enabled=wrapup_enabled, - wrapup_iteration=wrapup_iteration, + wrapup_iteration=design_wrapup_iteration, context_keep_tool_results=context_keep_tool_results, distill_enabled=distill_enabled, ) design_elapsed = time.time() - design_start design_analysis = design_result.get("analysis", "") - all_input_tokens += design_result.get("input_tokens", 0) - all_output_tokens += design_result.get("output_tokens", 0) - all_cost += design_result.get("cost", 0.0) + # Include the distillation pre-pass, which the loop reports separately + # from its own token/cost fields (resolve.py seeds totals the same way). + all_input_tokens += design_result.get("input_tokens", 0) + design_result.get("distill_input_tokens", 0) + all_output_tokens += design_result.get("output_tokens", 0) + design_result.get("distill_output_tokens", 0) + all_cost += design_result.get("cost", 0.0) + design_result.get("distill_cost", 0.0) if not design_analysis: post( @@ -1381,6 +1411,11 @@ def _run_single_review(council_model): "⚠️ **Agent loop blocked!** The revised design contained " "`/agent` command(s). Blocked for safety." ) + # Discard the contaminated revision, not just its posting — otherwise + # it still flows into Stages 3a/3b/3c and the workflow's Stage 4 + # (Stage 3c already nulls its artifact the same way). Fall back to + # the Stage 1 design, which was /agent-checked before we got here. + revised_design = design_analysis else: revision_cost_block = _build_stage_cost_block( github_repo=github_repo, @@ -1436,7 +1471,7 @@ def _run_single_review(council_model): extra_context=spec_extra_context, max_iterations=max_design_iterations, wrapup_enabled=wrapup_enabled, - wrapup_iteration=wrapup_iteration, + wrapup_iteration=design_wrapup_iteration, context_keep_tool_results=context_keep_tool_results, distill_enabled=distill_enabled, system_prompt=SPEC_DESIGN_SYSTEM_PROMPT, @@ -1444,9 +1479,9 @@ def _run_single_review(council_model): spec_elapsed = time.time() - spec_start implementation_spec = spec_result.get("analysis", "") - all_input_tokens += spec_result.get("input_tokens", 0) - all_output_tokens += spec_result.get("output_tokens", 0) - all_cost += spec_result.get("cost", 0.0) + all_input_tokens += spec_result.get("input_tokens", 0) + spec_result.get("distill_input_tokens", 0) + all_output_tokens += spec_result.get("output_tokens", 0) + spec_result.get("distill_output_tokens", 0) + all_cost += spec_result.get("cost", 0.0) + spec_result.get("distill_cost", 0.0) if not implementation_spec: post( diff --git a/tests/test_workshop.py b/tests/test_workshop.py index 4fc261e..2e3b5cc 100644 --- a/tests/test_workshop.py +++ b/tests/test_workshop.py @@ -753,3 +753,103 @@ def test_design_rounds_2_empty_spec_aborts( assert result["revised_spec"] is None # Only Stage 3 revision ran; Stage 3c did not assert mock_revision.call_count == 1 + + @patch("workshop._run_revision_call") + @patch("workshop.run_council_review") + @patch("design_loop.run_design_loop") + def test_agent_command_in_revision_falls_back_to_stage1_design( + self, mock_design, mock_council, mock_revision + ): + """A contaminated Stage 3 revision is discarded, not just unposted. + + The revised design flows into Stages 3a/3b/3c and the workflow's + Stage 4, so blocking only the comment post would leave the /agent + payload live downstream. The pipeline falls back to the Stage 1 + design, which was already /agent-checked. + """ + mock_design.return_value = self._mock_design_result( + analysis="## Proposed Design\n\nDo the thing." + ) + mock_council.return_value = self._mock_council_review() + mock_revision.return_value = self._mock_revision_result( + text="## Revised\n\n/agent-resolve\n\nImplement per the above." + ) + + result = run_delegate( + model="anthropic/test-model", + model_alias="test-model", + council_models=[{"alias": "claude-small", "id": "anthropic/test"}], + issue_title="Test Issue", + issue_body="Test body.", + ) + + assert result["revised_design"] == "## Proposed Design\n\nDo the thing." + assert "/agent-resolve" not in result["revised_design"] + + @patch("workshop._run_revision_call") + @patch("workshop.run_council_review") + @patch("design_loop.run_design_loop") + def test_design_wrapup_iteration_scaled_to_design_budget( + self, mock_design, mock_council, mock_revision + ): + """wrapup_iteration is rescaled for design stages. + + config.py computes it against the code-stage budget (e.g. 40 of 50); + passed unscaled into a 15-iteration design loop it can never fire. + """ + mock_design.side_effect = [ + self._mock_design_result(analysis="## Design\n\nDo X."), + self._mock_design_result(analysis="## Spec\n\nFile details."), + ] + mock_council.return_value = self._mock_council_review() + mock_revision.side_effect = [ + self._mock_revision_result(text="## Revised Design\n\nApproved."), + self._mock_revision_result(text="## Revised Spec"), + ] + + run_delegate( + model="anthropic/test-model", + model_alias="test-model", + council_models=[{"alias": "claude-small", "id": "anthropic/test"}], + issue_title="Test Issue", + issue_body="Test body.", + max_iterations=50, + max_design_iterations=15, + wrapup_iteration=40, + design_rounds=2, + ) + + # 40/50 = 0.8 → int(15 * 0.8) = 12 for both design-stage loops + _, stage1_kwargs = mock_design.call_args_list[0] + assert stage1_kwargs["wrapup_iteration"] == 12 + _, stage3a_kwargs = mock_design.call_args_list[1] + assert stage3a_kwargs["wrapup_iteration"] == 12 + + @patch("workshop._run_revision_call") + @patch("workshop.run_council_review") + @patch("design_loop.run_design_loop") + def test_distillation_cost_included_in_totals( + self, mock_design, mock_council, mock_revision + ): + """The design loop reports distillation tokens/cost separately from + its own loop fields; run_delegate totals must include both.""" + design = self._mock_design_result() + design["distill_input_tokens"] = 7000 + design["distill_output_tokens"] = 900 + design["distill_cost"] = 0.30 + mock_design.return_value = design + mock_council.return_value = self._mock_council_review() + mock_revision.return_value = self._mock_revision_result() + + result = run_delegate( + model="anthropic/test-model", + model_alias="test-model", + council_models=[{"alias": "claude-small", "id": "anthropic/test"}], + issue_title="Test Issue", + issue_body="Test body.", + ) + + # loop 1000 + distill 7000 + council 200 + revision 300 + assert result["total_input_tokens"] == 8500 + # loop 0.05 + distill 0.30 + council 0.01 + revision 0.02 + assert result["total_cost"] == pytest.approx(0.38) From df3fb9a803005e6d49b35f367f436b658d82b45c Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 17:40:08 +0000 Subject: [PATCH 24/39] design_loop.py: add moving-tail prompt-cache markers resolve.py and reconcile.py place a moving-tail cache_control marker on the last message before each API call (restored in #621 after the #606 regression); design_loop never got the fix, so standalone /agent-design and workshop/delegate design stages (Stage 1, Stage 3a) run at 0% cache hit on Anthropic models and pay full input price every iteration. Port the same strip-and-place logic, provider-gated with an explicit TTL for Gemini, matching the sibling loops verbatim. Closes the last Tier-3 sibling-drift item from the 2026-06-11 comprehensive review. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/design_loop.py | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/lib/design_loop.py b/lib/design_loop.py index 524d083..4caa0eb 100644 --- a/lib/design_loop.py +++ b/lib/design_loop.py @@ -388,6 +388,15 @@ def run_design_loop( {"role": "user", "content": user_content}, ] + # Cache strategy: moving-tail cache_control marker on the LAST message + # before each API call, same as resolve.py and reconcile.py (see + # resolve.py for the full rationale). Without it, design loops — and + # workshop/delegate Stage 1/3a, which run through here — pay full input + # price every iteration on Anthropic models. + _use_cache_markers = model.startswith(("anthropic/", "claude", "gemini/", "vertex_ai/")) + # Gemini's cache semantics differ from Anthropic; explicit TTL is required. + _cache_ttl = "3600s" if model.startswith(("gemini/", "vertex_ai/")) else None + total_input_tokens = 0 total_output_tokens = 0 total_cost = 0.0 @@ -401,6 +410,31 @@ def run_design_loop( for iteration in range(max_iterations): print(f"=== Iteration {iteration + 1}/{max_iterations} ===") + # Place the moving-tail cache_control marker on the last message. + # Strip every existing marker first to avoid accumulation past + # Anthropic's 4-marker limit. + if _use_cache_markers and messages: + for _msg in messages: + _c = _msg.get("content") + if isinstance(_c, list): + for _blk in _c: + if isinstance(_blk, dict): + _blk.pop("cache_control", None) + _tail = messages[-1] + _tail_content = _tail.get("content") + _cc: dict = {"type": "ephemeral"} + if _cache_ttl: + _cc["ttl"] = _cache_ttl + if isinstance(_tail_content, str): + _tail["content"] = [ + {"type": "text", "text": _tail_content, "cache_control": _cc} + ] + elif isinstance(_tail_content, list) and _tail_content: + # Apply the marker to the last block of the list. + _last_blk = _tail_content[-1] + if isinstance(_last_blk, dict): + _last_blk["cache_control"] = _cc + response = completion_with_retries( litellm_completion, model=model, From b9ca05c54b2f998358947882911acf2bf172cbfd Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sun, 5 Jul 2026 22:55:28 +0000 Subject: [PATCH 25/39] Unify wrapup-injection mechanics: every-iteration escalation in all loops MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three loops had drifted into three implementations of the same nudge: resolve re-injected every iteration past the threshold, reconcile injected exactly once (==), design_loop injected once via flag (>=). The 2026-06-11 comprehensive review flagged the divergence (SUMMARY.md Tier 3); resolve's repeated re-injection was confirmed as deliberate escalation — a single nudge gets buried under subsequent tool results. Unify all three on resolve's semantics: inject every iteration once iteration + 1 >= threshold. reconcile switches == to >=; design_loop drops its injected-once flag. Message content stays per-loop (each already recomputes the remaining-iteration count per injection). The deliberate-escalation rationale is now documented at all three sites. Behavior change: reconcile and design_loop agents past the threshold now receive the wrap-up nudge every iteration instead of once. With the moving-tail cache markers now in all three loops, the appended nudges are cheap. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- lib/design_loop.py | 7 ++++--- lib/reconcile.py | 6 +++++- lib/resolve.py | 3 +++ 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/lib/design_loop.py b/lib/design_loop.py index 4caa0eb..ec89fc4 100644 --- a/lib/design_loop.py +++ b/lib/design_loop.py @@ -404,7 +404,6 @@ def run_design_loop( total_cache_creation_tokens = 0 final_analysis = None last_response = None - wrapup_injected = False iteration = 0 for iteration in range(max_iterations): @@ -508,10 +507,13 @@ def run_design_loop( break # Graceful wrapup injection + # Re-injected EVERY iteration past the threshold — deliberate + # escalation, same mechanics as resolve.py and reconcile.py: one + # nudge gets buried under subsequent tool results, repeated + # pressure keeps the agent wrapping up. if ( wrapup_enabled and wrapup_iteration > 0 - and not wrapup_injected and iteration + 1 >= wrapup_iteration ): remaining = max_iterations - (iteration + 1) @@ -525,7 +527,6 @@ def run_design_loop( "Do not start new lines of inquiry." ), }) - wrapup_injected = True analysis = final_analysis or last_response or "" diff --git a/lib/reconcile.py b/lib/reconcile.py index 7a77d93..80e90b8 100644 --- a/lib/reconcile.py +++ b/lib/reconcile.py @@ -496,7 +496,11 @@ def main(): last_iteration = iteration print(f"=== Iteration {iteration + 1}/{MAX_ITERATIONS} ===") - if WRAPUP_ENABLED and WRAPUP_ITERATION > 0 and iteration + 1 == WRAPUP_ITERATION: + # Re-injected EVERY iteration past the threshold — deliberate + # escalation, same mechanics as resolve.py and design_loop: one + # nudge gets buried under subsequent tool results, repeated + # pressure keeps the agent wrapping up. + if WRAPUP_ENABLED and WRAPUP_ITERATION > 0 and iteration + 1 >= WRAPUP_ITERATION: remaining = MAX_ITERATIONS - WRAPUP_ITERATION print(f" [Wrapup] Injecting wrapup message at iteration {iteration + 1}") messages.append({ diff --git a/lib/resolve.py b/lib/resolve.py index 91e8c6e..ae0700a 100644 --- a/lib/resolve.py +++ b/lib/resolve.py @@ -1339,6 +1339,9 @@ def main(): # Inject live wrapup message when the threshold is reached. # This is more effective than the system prompt hint alone — the agent # is deep in context by this point and needs a fresh, visible reminder. + # Re-injected EVERY iteration past the threshold — deliberate + # escalation (one nudge gets buried under subsequent tool results); + # reconcile.py and design_loop use the same mechanics. if WRAPUP_ENABLED and WRAPUP_ITERATION > 0 and iteration + 1 >= WRAPUP_ITERATION: remaining = MAX_ITERATIONS - (iteration + 1) is_final = (iteration + 1 == MAX_ITERATIONS) From f0f55cf7bb6af8cd52d2b5920c1a0d4cb4ef0b45 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Mon, 6 Jul 2026 01:50:13 +0000 Subject: [PATCH 26/39] Correct stale "revision plan" descriptions of delegate Stage 6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stage 6 has been an agentic code revision since #521 (April 2026): the workflow re-invokes resolve.py on the PR branch with the Stage 5 council reviews and design/spec artifacts injected via EXTRA_FILES. Three stale references still described the pre-#521 one-shot "revision plan": run_delegate's docstring, a nearby comment, and the delegate result comment's stage checklist. The stale docstring also misled the 2026-06-11 comprehensive review into calling the council loop open and ranking "close the council loop" as the top feature — add CORRECTIONS.md to the review directory recording the error and what survives it, with a pointer at the top of SUMMARY.md. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .github/workflows/remote-dev-bot.yml | 2 +- .../2026-06-11/CORRECTIONS.md | 39 +++++++++++++++++++ comprehensive-review/2026-06-11/SUMMARY.md | 4 ++ lib/workshop.py | 7 +++- 4 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 comprehensive-review/2026-06-11/CORRECTIONS.md diff --git a/.github/workflows/remote-dev-bot.yml b/.github/workflows/remote-dev-bot.yml index e6da516..d7f01c3 100644 --- a/.github/workflows/remote-dev-bot.yml +++ b/.github/workflows/remote-dev-bot.yml @@ -4298,7 +4298,7 @@ jobs: echo "3. ✅ Design revision" echo "4. ✅ Implementation (resolve)" echo "5. ✅ Council code review" - echo "6. ✅ Code revision plan" + echo "6. ✅ Agentic code revision" echo "" echo "See the [workflow run logs]($RUN_URL) for full details." } > /tmp/delegate_result_comment.md diff --git a/comprehensive-review/2026-06-11/CORRECTIONS.md b/comprehensive-review/2026-06-11/CORRECTIONS.md new file mode 100644 index 0000000..d485c5a --- /dev/null +++ b/comprehensive-review/2026-06-11/CORRECTIONS.md @@ -0,0 +1,39 @@ +# Corrections to the 2026-06-11 review + +Discovered during the fix campaign; the findings files are preserved as +written, with errors noted here instead. + +## Delegate Stage 6 is agentic, not advisory (§4.1, findings-prompts.md §5) + +The review claims delegate Stage 6 "posts a 'revision plan' comment; no +stage *acts*" on council findings, and ranks "close the council loop" as +the highest-leverage feature on that basis. + +**This was already false when the review was written.** PR #521 +(`c13f7f8`, merged to dev 2026-04-11 and on main before the May usage +window) replaced the one-shot revision-plan call with an agentic Stage 6: +the workflow re-invokes `resolve.py` against the Stage 4 PR branch with +the Stage 5 council reviews, the revised design, and the revised spec +injected via `EXTRA_FILES`; the agent applies what it judges worthwhile +and commits to the same branch, bounded by `delegate_max_iterations` and +the watchdog timeout. + +The error most likely came from `run_delegate`'s then-stale docstring +("6. Code revision plan — describes fixes"), which described the pre-#521 +design. The docstring and two related comments were fixed alongside this +note. + +Two review claims survive the correction: + +- The BT-MM methodology catch on bridge-analysis PR #438 genuinely went + unactioned — but that council was a manually invoked + `/agent-review council=true`, a path that has no revision stage. The + open loop is in standalone review (and build Stage 2), not delegate. + Note `resolve.py` on a PR already ingests PR comments, so commenting + `/agent-resolve` on a reviewed PR closes that loop manually today. +- Delegate's closed loop has essentially no production validation: the + only May delegate run predates the #628/#631-633 prompt fixes. Code + review of the Stage 6 step (2026-07-06) found the plumbing correct + (env complete for resolve.py's `_require_env` set, wrapup consistent + with the code budget, skip/timeout/cost-merge paths sane); it awaits a + real run. diff --git a/comprehensive-review/2026-06-11/SUMMARY.md b/comprehensive-review/2026-06-11/SUMMARY.md index b2f1d98..42a7bd8 100644 --- a/comprehensive-review/2026-06-11/SUMMARY.md +++ b/comprehensive-review/2026-06-11/SUMMARY.md @@ -1,5 +1,9 @@ # remote-dev-bot — Comprehensive Review (2026-06-11) +> **Correction (2026-07-06):** §4.1's claim that delegate Stage 6 is an +> advisory "revision plan" is wrong — Stage 6 has been an agentic revision +> pass since April (#521). See [CORRECTIONS.md](CORRECTIONS.md). + Written 2026-06-11 against dev @ de72856 (PRs #631-633 merged). Synthesizes five phases of incremental findings written by sequential subagents and preserved alongside this file: `findings-{code,workflows,docs,usage,prompts}.md`. diff --git a/lib/workshop.py b/lib/workshop.py index 7203adc..cd9c8c1 100644 --- a/lib/workshop.py +++ b/lib/workshop.py @@ -1117,7 +1117,9 @@ def run_delegate( 3c. Spec revision (one-shot) — conditional on design_rounds >= 2 4. Implementation / resolve (agentic, same as /agent-resolve) 5. Council code review (parallel, same as /agent-build Stage 2) - 6. Code revision plan — main agent reads code reviews, describes fixes + 6. Agentic code revision — the workflow re-invokes resolve.py on the + PR branch with the Stage 5 reviews (plus revised design/spec) + injected via EXTRA_FILES; the agent applies the fixes and commits Parameters ---------- @@ -1713,7 +1715,8 @@ def _run_single_spec_review(council_model): # - Run Stages 1-3 (and optionally 3a-3c) here # - Pass the revised design (and revised spec) to the resolve step # - After resolve creates a PR, run Stage 5 (council code review) - # - Post Stage 6 (code revision plan) as a comment + # - Run Stage 6 (agentic code revision): resolve.py on the PR branch + # with the Stage 5 reviews injected via EXTRA_FILES stages_done_label = "Stages 1-3c" if design_rounds >= 2 else "Stages 1-3" post( From 05dd821c523f65381346d448696ecc182c9367e5 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Mon, 6 Jul 2026 16:19:03 +0000 Subject: [PATCH 27/39] e2e.sh: count reconcile failures in the exit code and summary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reconcile e2e test's pass/fail was verified thoroughly (force-push SHA change + "Reconcile complete" comment) and printed in the results table, but RECONCILE_FAIL was never added to total_fail and had no summary line — a reconcile failure exited 0 and the suite read as green. From the 2026-06-11 comprehensive review (findings-workflows top finding 7, first item). The other two accounting items in that finding (rf-feedback sub-outcome not counted; duplicated wrapup run) are left as-is for now. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- tests/e2e.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/e2e.sh b/tests/e2e.sh index 4331471..82693ce 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -1351,6 +1351,7 @@ log " Total cost: \$$total_cost ($cost_count tests with cost data)" log "" log "=========================================" log " Smoke tests: Pass: $pass Fail: $fail Timeout: $timeout_count" +log " Reconcile test: Pass: $RECONCILE_PASS Fail: $RECONCILE_FAIL" log " Review+Feedback: Pass: $RF_PASS Fail: $RF_FAIL" log " Timeout test: Pass: $TIMEOUT_PHASE_PASS Fail: $TIMEOUT_PHASE_FAIL" log " Wrapup test: Pass: $WRAPUP_PHASE_PASS Fail: $WRAPUP_PHASE_FAIL" @@ -1358,7 +1359,7 @@ log " Total cost: \$$total_cost" log "=========================================" # Exit with failure if any test didn't pass -total_fail=$((fail + timeout_count + RF_FAIL + TIMEOUT_PHASE_FAIL + WRAPUP_PHASE_FAIL)) +total_fail=$((fail + timeout_count + RECONCILE_FAIL + RF_FAIL + TIMEOUT_PHASE_FAIL + WRAPUP_PHASE_FAIL)) if [[ $total_fail -gt 0 ]]; then exit 1 fi From e744adcece3dfc8dd4951172af6aef2abe68ac69 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Mon, 20 Jul 2026 05:23:41 +0000 Subject: [PATCH 28/39] Model aliases: three capability tiers (small/medium/large), current models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-1.0 breaking change to the alias convention, made now while there are no external users to break. Aliases become three capability tiers per provider — small (value coding workhorse), medium (provider flagship), large (frontier above the flagship, where one exists) — with the reasoning documented in the config: - Tiers anchor on capability at agentic coding, not price; the ladder is monotone (up a tier is never weaker, never cheaper). - Every alias always resolves to a runnable model: providers lacking a distinct model for a tier co-point at the tier below (visible ladder compression, not a hole, and no surprise-pricing or can't-code traps). - The alias set is append-only post-1.0; repointing is expected and gets a changelog entry. Users who would draw the lines differently define their own aliases in their repo config. New mappings: claude-small=sonnet-5, claude-medium=opus-4-8 (was claude-large), claude-large=fable-5 (new frontier tier), gpt-small=gpt-5.3-codex, gpt-medium=gpt-5.6-sol (gpt-large co-points — no distinct OpenAI frontier tier suits agentic coding today), gemini-small/medium/large=gemini-3.5-flash (co-pointed; medium repoints to Gemini 3.5 Pro at GA). Docs updated to the tier language (README tier table + rationale, how-it-works, onboarding, install, demo); default escalation advice now points at *-medium. Everything load-bearing (auto default_model, councils, e2e smoke tests) references *-small, which did not move. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 48 +++++++++++++++++++++++---------- demo.md | 4 +-- how-it-works.md | 12 ++++----- install.md | 2 +- onboarding.md | 11 ++++++-- remote-dev-bot.yaml | 65 ++++++++++++++++++++++++++++++++++++--------- 6 files changed, 104 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index c5fed8d..e65752a 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ to design agents that can autonomously handle real development tasks. ## How It Works 1. Create a GitHub issue describing a feature or bug -2. Comment `/agent-resolve` (or `/agent-resolve-claude-large`, etc.) to trigger +2. Comment `/agent-resolve` (or `/agent-resolve-claude-medium`, etc.) to trigger implementation 3. A GitHub Action spins up an AI agent that runs a custom LiteLLM agent loop that: @@ -65,11 +65,11 @@ comment (no code changes). | Command | What it does | | ------------------------------- | ---------------------------------------------------- | | `/agent-resolve` | Resolve the issue and open a PR (default model) | -| `/agent-resolve-claude-large` | Resolve with a specific model | +| `/agent-resolve-claude-medium` | Resolve with a specific model | | `/agent-design` | Explore codebase and post a design analysis comment | -| `/agent-design-claude-large` | Design analysis with a specific model | +| `/agent-design-claude-medium` | Design analysis with a specific model | | `/agent-review` | Post a code review comment on a PR (no code changes) | -| `/agent-review-claude-large` | Code review with a specific model | +| `/agent-review-claude-medium` | Code review with a specific model | | `/agent-workshop[-<model>]` | Design analysis + multi-model council critique | | `/agent-build[-<model>]` | Implement issue + multi-model council code review | | `/agent-delegate[-<model>]` | Full design-to-implementation cycle with no human pauses | @@ -78,8 +78,8 @@ comment (no code changes). Modes and model aliases are configured in `remote-dev-bot.yaml`. **Mobile-friendly syntax:** Commands are case-insensitive and you can use spaces -instead of dashes: `/agent resolve claude large` works the same as -`/agent-resolve-claude-large`. +instead of dashes: `/agent resolve claude medium` works the same as +`/agent-resolve-claude-medium`. ### Per-Invocation Arguments @@ -113,13 +113,30 @@ Model aliases (like `claude-small`) map to **LiteLLM model identifiers** in `remote-dev-bot.yaml`. Remote Dev Bot uses LiteLLM to talk to different LLM providers through a unified interface. +Aliases come in three capability tiers per provider, so you never have to +track model releases: + +| Tier | Meaning | Typical use | +| ---------- | ---------------------------------------------------- | ----------- | +| `*-small` | Best value-for-money coding workhorse | The default; most tasks | +| `*-medium` | The provider's flagship | Complex multi-file features | +| `*-large` | Frontier model above the flagship, where one exists | Hardest long-horizon work | + +Tiers are anchored on capability at agentic coding, not price, and the ladder +is monotone — going up a tier never gets you a weaker model. Where a provider +has no distinct model for a tier, the alias points at the tier below it, so +every alias always runs. The model landscape doesn't always divide cleanly +into three tiers; these mappings are judgment calls, and you can always +define your own aliases (see "Adding Models" below) if you'd draw the lines +differently. An abstraction doesn't have to be perfect to be useful. + **Model ID format:** `provider/model-name` | Provider | Prefix | Example | | ------------------ | ------------ | ----------------------------- | -| Anthropic (Claude) | `anthropic/` | `anthropic/claude-sonnet-4-5` | -| OpenAI (GPT) | `openai/` | `openai/gpt-5.1-codex-mini` | -| Google (Gemini) | `gemini/` | `gemini/gemini-2.5-flash` | +| Anthropic (Claude) | `anthropic/` | `anthropic/claude-sonnet-5` | +| OpenAI (GPT) | `openai/` | `openai/gpt-5.3-codex` | +| Google (Gemini) | `gemini/` | `gemini/gemini-3.5-flash` | ### Supported Providers @@ -161,12 +178,15 @@ Prefix the model string with the provider name in `remote-dev-bot.yaml` (e.g., **For most tasks:** Use the default (`/agent-resolve`). Claude Sonnet (`claude-small`) offers a good balance of capability and cost. -**For complex multi-file features:** Use `/agent-resolve-claude-large` (Opus) or -`/agent-resolve-gpt-large` (GPT Codex). These models handle larger contexts and -more intricate reasoning. +**For complex multi-file features:** Use `/agent-resolve-claude-medium` (Opus) +or `/agent-resolve-gpt-medium`. Flagship models handle larger contexts and more +intricate reasoning. + +**For the hardest long-horizon work:** `/agent-resolve-claude-large` (Fable) +is the frontier tier — expect roughly 2-4x the cost of a medium run. **For coding-heavy tasks:** Models with "codex" in the name (e.g., -`openai/gpt-5.1-codex-mini`) are specifically tuned for code generation and may +`openai/gpt-5.3-codex`) are specifically tuned for code generation and may perform better on implementation tasks. ### Commit Trailers @@ -435,7 +455,7 @@ The agent ran, posted a comment with its evaluation, but didn't open a PR. This means the agent judged that it couldn't fully resolve the issue — it hit the iteration limit, got confused, or determined its changes were incomplete. -Try a more capable model (`/agent-resolve-claude-large`) or add more detail to +Try a more capable model (`/agent-resolve-claude-medium`) or add more detail to the issue description. The agent's evaluation comment will say what it attempted and why it stopped. diff --git a/demo.md b/demo.md index 500f30d..a9bd9c6 100644 --- a/demo.md +++ b/demo.md @@ -121,8 +121,8 @@ This is the normal code review workflow — just with an AI making the fixes. | **Design** | `/agent-design` | Analyzes the problem and posts a comment | | **Review** | `/agent-review` | Reviews a PR and posts feedback | -All modes support model variants: `/agent-resolve-claude-large`, -`/agent-design-gpt-large`, etc. +All modes support model variants: `/agent-resolve-claude-medium`, +`/agent-design-gemini-small`, etc. --- diff --git a/how-it-works.md b/how-it-works.md index 2f9f774..3f5f258 100644 --- a/how-it-works.md +++ b/how-it-works.md @@ -50,7 +50,7 @@ This is the "engine" — the shared infrastructure that all target repos use. | File | Purpose | |------|---------| | `.github/workflows/remote-dev-bot.yml` | The reusable workflow. Contains all the logic: parses model aliases, runs the LiteLLM agent loop, resolves issues, creates PRs. Target repos call this. | -| `remote-dev-bot.yaml` | Base configuration. Defines model aliases (`claude-small`, `claude-large`, etc.) and agent settings (max iterations, PR type). | +| `remote-dev-bot.yaml` | Base configuration. Defines model aliases (`claude-small`, `claude-medium`, etc.) and agent settings (max iterations, PR type). | | `.github/workflows/agent.yml` | Shim workflow. Also serves as the template — copy this to target repos. | | `install.md` | Step-by-step setup instructions for humans or AI assistants. | @@ -81,13 +81,13 @@ This is where you want the AI agent to help with development. ## How the Pieces Connect -When someone comments `/agent-resolve-claude-large` on an issue: +When someone comments `/agent-resolve-claude-medium` on an issue: 1. **Shim triggers** — The target repo's `agent.yml` fires on the comment 2. **Calls reusable workflow** — The shim calls `remote-dev-bot.yml@main` from remote-dev-bot 3. **Config checkout** — remote-dev-bot.yml sparse-checks out `remote-dev-bot.yaml` and `lib/` from remote-dev-bot 4. **Config merge** — base config from remote-dev-bot is merged with any override config in the target repo -5. **Model resolution** — The alias `claude-large` is resolved to a model ID like `anthropic/claude-opus-4-5` +5. **Model resolution** — The alias `claude-medium` is resolved to a model ID like `anthropic/claude-opus-4-8` 6. **Feedback** — A rocket emoji is added to your comment and you're assigned to the issue, so you can see at a glance which issues have active work 7. **Agent runs** — LiteLLM agent loop (resolve.py) reads the issue, explores the codebase, makes changes 8. **PR created** — A draft (or ready) PR is opened with the changes @@ -95,7 +95,7 @@ When someone comments `/agent-resolve-claude-large` on an issue: For `/agent-workshop` and `/agent-build`, steps 1–6 are identical. Step 7 runs the same agent loop (design or resolve), then `lib/workshop.py` runs each council model in sequence and posts a review comment. The bot pauses there; no further automation happens until a human triggers the next command. ``` -User comments /agent-resolve-claude-large +User comments /agent-resolve-claude-medium │ ▼ ┌─────────────────────┐ @@ -137,12 +137,12 @@ agent: pr_type: ready # target-repo/remote-dev-bot.yaml (override) -default_model: claude-large +default_model: claude-medium agent: max_iterations: 30 ``` -Result: `default_model: claude-large`, `max_iterations: 30`, `pr_type: ready` +Result: `default_model: claude-medium`, `max_iterations: 30`, `pr_type: ready` (inherited from base). Merges are deep (leaf-level): overriding `agent.max_iterations` does not diff --git a/install.md b/install.md index 76ccdbd..6053487 100644 --- a/install.md +++ b/install.md @@ -988,7 +988,7 @@ different repo owners. See the shim template in `.github/workflows/agent.yml`. ### Agent runs but hits max iterations - The agent completed the work but couldn't gracefully stop -- Try a more capable model: `/agent-resolve-claude-large` +- Try a more capable model: `/agent-resolve-claude-medium` - Or increase `max_iterations` in `remote-dev-bot.yaml` ### Agent runs but skips PR creation diff --git a/onboarding.md b/onboarding.md index cf53fc3..82f9003 100644 --- a/onboarding.md +++ b/onboarding.md @@ -235,17 +235,24 @@ When you run `/agent-resolve` without a model suffix, you get the default model ### When to Use Larger Models -Try `/agent-resolve-claude-large` or `/agent-resolve-gpt-large` when: +Model aliases come in three tiers: `*-small` (value workhorse, the default), +`*-medium` (the provider's flagship), and `*-large` (frontier, where the +provider offers one — expect a multiple of medium's cost). + +Try `/agent-resolve-claude-medium` or `/agent-resolve-gpt-medium` when: - The task involves complex reasoning across multiple files - Previous attempts with the default model failed or produced poor results - You're working on architecture-level changes +For the hardest long-horizon tasks, `/agent-resolve-claude-large` escalates +to the frontier tier. + ### Try It 1. Create an issue with a moderately complex task 2. First try: `/agent-resolve` (default model) -3. If the result isn't satisfactory, try: `/agent-resolve-claude-large` +3. If the result isn't satisfactory, try: `/agent-resolve-claude-medium` 4. Compare the results and costs ### Cost Tradeoffs diff --git a/remote-dev-bot.yaml b/remote-dev-bot.yaml index cb8995f..baba8b0 100644 --- a/remote-dev-bot.yaml +++ b/remote-dev-bot.yaml @@ -9,7 +9,7 @@ # /agent-delegate — full design-to-implementation cycle with no human pauses # /agent-reconcile — rebase PR onto base branch and resolve conflicts # -# To use a specific model, append it: /agent-resolve-claude-large +# To use a specific model, append it: /agent-resolve-claude-medium # This works for any mode. See models: below for available aliases. # auto = pick the small model of whichever provider API key is configured, @@ -126,30 +126,69 @@ modes: - gpt-small - gemini-small +# Model aliases — the tier system +# +# Aliases exist so commands and configs don't have to track model releases. +# Three capability tiers per provider: +# +# *-small best value-for-money agentic-coding workhorse (~sonnet class) +# *-medium the provider's flagship (~opus class) +# *-large frontier model above the flagship, where one exists +# +# Rules used when repointing (models change; alias names don't): +# - Tiers are anchored on CAPABILITY at agentic coding, not price. Price +# is an expectation each tier carries, not the admission test. +# - The ladder is monotone: going up a tier never gets a weaker model, +# and never a cheaper one. +# - Every alias always resolves to a runnable model. Where a provider has +# no distinct model for a tier, the alias co-points at the tier below — +# visible ladder compression rather than a hole. (Tempting alternatives +# rejected: pointing tiers at models that can't do nontrivial agentic +# coding, or at models with surprise 3x+ pricing.) +# - The alias set is append-only; repointing is expected and gets a +# changelog entry, since it changes what your runs cost. +# +# The model landscape doesn't always divide cleanly into three tiers — +# these mappings involve judgment calls and reasonable people may disagree +# with them. If you do: define your own aliases in your repo's +# remote-dev-bot.yaml (they merge with this list) and pin any model you +# like. An abstraction doesn't have to be perfect to be useful. models: claude-small: - id: anthropic/claude-sonnet-4-6 - description: "Balanced cost/capability — good default for most tasks" + id: anthropic/claude-sonnet-5 + description: "Anthropic workhorse — best value for most tasks" + + claude-medium: + id: anthropic/claude-opus-4-8 + description: "Anthropic flagship — complex multi-file features" claude-large: - id: anthropic/claude-opus-4-7 - description: "Most capable — for complex multi-file features" + id: anthropic/claude-fable-5 + description: "Anthropic frontier — hardest long-horizon work (~2-4x medium cost)" gpt-small: - id: openai/gpt-5.1-codex-mini - description: "OpenAI GPT-5.1 Codex Mini — fast, cost-effective coding" + id: openai/gpt-5.3-codex + description: "OpenAI agentic-coding workhorse (Codex line)" + + gpt-medium: + id: openai/gpt-5.6-sol + description: "OpenAI flagship" gpt-large: - id: openai/gpt-5.5 - description: "OpenAI GPT-5.5 — frontier coding + reasoning" + id: openai/gpt-5.6-sol + description: "Co-pointed at gpt-medium — no distinct OpenAI frontier tier suits agentic coding today" gemini-small: - id: gemini/gemini-2.5-flash - description: "Google Gemini 2.5 Flash — good default" + id: gemini/gemini-3.5-flash + description: "Google workhorse — strongest Gemini for agentic coding today" + + gemini-medium: + id: gemini/gemini-3.5-flash + description: "Co-pointed at gemini-small — repoint to Gemini 3.5 Pro when it reaches GA" gemini-large: - id: gemini/gemini-2.5-pro - description: "Google Gemini 2.5 Pro" + id: gemini/gemini-3.5-flash + description: "Co-pointed — no distinct Google frontier tier today" # Agent settings — apply to all modes unless overridden per-mode above. # Note: 'openhands:' is accepted as a backward-compatible alias for 'agent:' From f0a11b170383b7e2891c1f748b64491214d08b8e Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Wed, 5 Aug 2026 16:57:34 +0000 Subject: [PATCH 29/39] Refresh alias targets for early-August model releases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two weeks passed between cutting this PR and merging it, and the market moved — which is the tier system's first live exercise, absorbed with zero alias-name changes: - claude-medium: opus-4-8 -> opus-5 (released Jul 24; same $5/$25 band, agentic-coding focused — a textbook flagship repoint) - gemini-small/medium/large: 3.5-flash -> 3.6-flash (stable, better at code/agentic AND cheaper — dominates, so the co-pointed trio moves together; Gemini 3.5 Pro slipped again, so medium stays co-pointed) Model IDs verified against the Anthropic models overview and the Gemini API docs. OpenAI lineup unchanged since the 5.6 family. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- README.md | 2 +- remote-dev-bot.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e65752a..f2faef6 100644 --- a/README.md +++ b/README.md @@ -136,7 +136,7 @@ differently. An abstraction doesn't have to be perfect to be useful. | ------------------ | ------------ | ----------------------------- | | Anthropic (Claude) | `anthropic/` | `anthropic/claude-sonnet-5` | | OpenAI (GPT) | `openai/` | `openai/gpt-5.3-codex` | -| Google (Gemini) | `gemini/` | `gemini/gemini-3.5-flash` | +| Google (Gemini) | `gemini/` | `gemini/gemini-3.6-flash` | ### Supported Providers diff --git a/remote-dev-bot.yaml b/remote-dev-bot.yaml index baba8b0..51b1e96 100644 --- a/remote-dev-bot.yaml +++ b/remote-dev-bot.yaml @@ -159,7 +159,7 @@ models: description: "Anthropic workhorse — best value for most tasks" claude-medium: - id: anthropic/claude-opus-4-8 + id: anthropic/claude-opus-5 description: "Anthropic flagship — complex multi-file features" claude-large: @@ -179,15 +179,15 @@ models: description: "Co-pointed at gpt-medium — no distinct OpenAI frontier tier suits agentic coding today" gemini-small: - id: gemini/gemini-3.5-flash + id: gemini/gemini-3.6-flash description: "Google workhorse — strongest Gemini for agentic coding today" gemini-medium: - id: gemini/gemini-3.5-flash - description: "Co-pointed at gemini-small — repoint to Gemini 3.5 Pro when it reaches GA" + id: gemini/gemini-3.6-flash + description: "Co-pointed at gemini-small — repoint to Gemini 3.5 Pro if/when it reaches GA" gemini-large: - id: gemini/gemini-3.5-flash + id: gemini/gemini-3.6-flash description: "Co-pointed — no distinct Google frontier tier today" # Agent settings — apply to all modes unless overridden per-mode above. From 1059cc9d664b4cd3c136ea5ef85977117966880f Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Wed, 5 Aug 2026 19:24:18 +0000 Subject: [PATCH 30/39] e2e: fail fast with a clear message when GH_TOKEN is dead The 2026-08-05 full-test-suite run failed before any LLM call because RDB_TESTER_PAT_TOKEN (set Feb 25) had expired: every gh call returned HTTP 401, the quota check spewed a bash syntax error trying to compare a JSON error body against an integer, and the failure surfaced as three confusing per-test errors instead of one diagnosis. Both e2e scripts now verify the token up front (gh api user) and exit with rotation instructions when it's invalid, and the shared check_graphql_quota copies guard against non-numeric parses. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- tests/e2e-security.sh | 15 +++++++++++++++ tests/e2e.sh | 15 +++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/tests/e2e-security.sh b/tests/e2e-security.sh index dba73e6..ab6c03d 100755 --- a/tests/e2e-security.sh +++ b/tests/e2e-security.sh @@ -58,6 +58,10 @@ check_graphql_quota() { fi remaining=$(echo "$result" | cut -d' ' -f1) reset_ts=$(echo "$result" | cut -d' ' -f2) + if ! [[ "$remaining" =~ ^[0-9]+$ ]]; then + echo "==> Warning: could not parse GraphQL rate limit response — proceeding anyway" + return + fi reset_time=$(python3 -c " import datetime ts = ${reset_ts} @@ -74,6 +78,17 @@ print(f'{local_t.strftime(\"%H:%M:%S\")} local / {utc_t.strftime(\"%H:%M:%S\")} echo "==> GraphQL quota: ${remaining} points remaining (resets ${reset_time})." } +# Fail fast with a clear message when GH_TOKEN is dead — an expired +# RDB_TESTER_PAT_TOKEN otherwise surfaces as confusing 401s on every +# issue-creation call after the suite has already started. +gh_login=$(gh api user --jq .login 2>/dev/null) || { + echo "ERROR: GH_TOKEN is invalid or expired (gh api user failed)." >&2 + echo "ERROR: In CI this token comes from secrets.RDB_TESTER_PAT_TOKEN —" >&2 + echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 + exit 1 +} +echo "==> Authenticated to GitHub as: ${gh_login}" + check_graphql_quota # --- Helpers --- diff --git a/tests/e2e.sh b/tests/e2e.sh index 4331471..250dabc 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -54,6 +54,10 @@ check_graphql_quota() { fi remaining=$(echo "$result" | cut -d' ' -f1) reset_ts=$(echo "$result" | cut -d' ' -f2) + if ! [[ "$remaining" =~ ^[0-9]+$ ]]; then + echo "==> Warning: could not parse GraphQL rate limit response — proceeding anyway" + return + fi reset_time=$(python3 -c " import datetime ts = ${reset_ts} @@ -70,6 +74,17 @@ print(f'{local_t.strftime(\"%H:%M:%S\")} local / {utc_t.strftime(\"%H:%M:%S\")} echo "==> GraphQL quota: ${remaining} points remaining (resets ${reset_time})." } +# Fail fast with a clear message when GH_TOKEN is dead — an expired +# RDB_TESTER_PAT_TOKEN otherwise surfaces as confusing 401s on every +# issue-creation call after the suite has already started. +gh_login=$(gh api user --jq .login 2>/dev/null) || { + echo "ERROR: GH_TOKEN is invalid or expired (gh api user failed)." >&2 + echo "ERROR: In CI this token comes from secrets.RDB_TESTER_PAT_TOKEN —" >&2 + echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 + exit 1 +} +echo "==> Authenticated to GitHub as: ${gh_login}" + check_graphql_quota # --- Test case definitions --- From f22ccdb417f057011846756f247d707ba6e642f0 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sat, 8 Aug 2026 04:49:00 +0000 Subject: [PATCH 31/39] CONTRIBUTING: document that RDB_TESTER_PAT_TOKEN is a gnovak PAT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The account table said the token belongs to the remote-dev-bot account, but that account was flagged by GitHub as a bot and authorized-trigger duty moved to gnovak — every e2e test issue on rdb-test is authored by gnovak. Documented the actual state, recommended a fine-grained PAT scoped to the test repo with the permissions the suite really needs, and noted the expiry failure mode (the 2026-08-05 suite failure). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- CONTRIBUTING.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b4968bb..1df1aaa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,21 +24,25 @@ Three separate GitHub identities are used so each role is cleanly separated: | Account | Purpose | Credentials stored as | | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | | `gnovak` | Repo owner. Used for normal development and testing. | `RDB_PAT_TOKEN` (on both repos), or GitHub App (`RDB_APP_ID` variable + `RDB_APP_PRIVATE_KEY` secret) | -| `remote-dev-bot` | Dedicated bot account. Collaborator on `remote-dev-bot-test`. Posts authorized test comments that trigger agent runs without attributing activity to `gnovak`. | `RDB_TESTER_PAT_TOKEN` (on remote-dev-bot) | +| `remote-dev-bot` | Dedicated bot account, collaborator on `remote-dev-bot-test`. **Historical:** it was flagged by GitHub as a bot account, so in practice `RDB_TESTER_PAT_TOKEN` is a `gnovak` PAT (verifiable: e2e test issues are authored by `gnovak`). | (none currently) | | `remote-dev-bot-tester` | Simulates an unauthorized external user. NOT a collaborator on any repo. | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` (on remote-dev-bot) | -### remote-dev-bot (bot account) details +### RDB_TESTER_PAT_TOKEN details -- Classic PAT with `public_repo` + `workflow` scopes, no expiration -- **`public_repo`** — create issues, post comments, open PRs in public repos - (standard e2e test flow) -- **`workflow`** — read/write `.github/workflows/` files; required when the - agent itself needs to modify workflow files in a PR (e.g., the dogfood use - case where rdb is editing its own workflow files). Without this scope, - writes to `.github/workflows/` fail with HTTP 404. -- Must be a collaborator on `remote-dev-bot-test` so the security gate allows - its trigger comments -- Keeps test activity out of `gnovak`'s GitHub contribution stats +In practice a PAT on the `gnovak` account (the dedicated `remote-dev-bot` +account was flagged by GitHub as a bot, so authorized-trigger duty moved to +the owner account; test activity therefore does show up in `gnovak`'s +contribution stats). + +- Recommended: fine-grained PAT scoped to `gnovak/remote-dev-bot-test` only, + with Contents read/write, Issues read/write, Pull requests read/write, and + Actions read/write (the suite monitors and cancels workflow runs). + A classic PAT with `repo` scope also works but grants far more than needed. +- Prefer a long or no expiration, and note the date somewhere visible — an + expired token surfaces as the e2e pre-flight failing with 401 (the + 2026-08-05 full-suite failure was exactly this, from a 90-day default). +- The token owner must pass the security gate on `remote-dev-bot-test` + (owner or collaborator) so its trigger comments start runs. ### remote-dev-bot-tester details @@ -81,7 +85,7 @@ Secrets stored on `gnovak/remote-dev-bot`: | `OPENAI_API_KEY` | OpenAI API key (for GPT models) | | `GEMINI_API_KEY` | Google AI API key (for Gemini models) | | `RDB_APP_PRIVATE_KEY` | GitHub App private key, for bot identity on comments/PRs. | -| `RDB_TESTER_PAT_TOKEN` | PAT for `remote-dev-bot` account (collaborator on rdb-test). Used by e2e tests to post authorized trigger comments. | +| `RDB_TESTER_PAT_TOKEN` | PAT for an account authorized on rdb-test (in practice `gnovak` — see account table). Used by e2e tests to create issues and post authorized trigger comments. | | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` | PAT for `remote-dev-bot-tester` (not a collaborator). Used by security e2e tests to verify unauthorized users are blocked. | Variables stored on `gnovak/remote-dev-bot`: From a311854718cf4b0fe2e3b42e0c6acdffe1c8e98a Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sat, 8 Aug 2026 04:55:54 +0000 Subject: [PATCH 32/39] Rework token docs to intent-first; pre-flight prints expiry, checks both tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous commit documented the gnovak-PAT workaround as the standing setup. The bot-account arrangement is the intended design — the account is just flagged by GitHub right now (its profile 404s on the public API) pending a support conversation. Restore the intended setup as primary in CONTRIBUTING with a clearly-marked temporary-state note carrying the stopgap guidance and the restore path. Pre-flight hardening while here: print the token's expiration date (GitHub-Authentication-Token-Expiration header) alongside the login so future runs self-document how long the credential has left, and validate UNAUTHORIZED_PAT up front in e2e-security.sh — its failures otherwise surface only mid-gating-test. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- CONTRIBUTING.md | 45 ++++++++++++++++++++++++++----------------- tests/e2e-security.sh | 16 ++++++++++++++- tests/e2e.sh | 3 ++- 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1df1aaa..969f1e6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,25 +24,34 @@ Three separate GitHub identities are used so each role is cleanly separated: | Account | Purpose | Credentials stored as | | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | | `gnovak` | Repo owner. Used for normal development and testing. | `RDB_PAT_TOKEN` (on both repos), or GitHub App (`RDB_APP_ID` variable + `RDB_APP_PRIVATE_KEY` secret) | -| `remote-dev-bot` | Dedicated bot account, collaborator on `remote-dev-bot-test`. **Historical:** it was flagged by GitHub as a bot account, so in practice `RDB_TESTER_PAT_TOKEN` is a `gnovak` PAT (verifiable: e2e test issues are authored by `gnovak`). | (none currently) | +| `remote-dev-bot` | Dedicated bot account. Collaborator on `remote-dev-bot-test`. Posts authorized test comments that trigger agent runs without attributing activity to `gnovak`. | `RDB_TESTER_PAT_TOKEN` (on remote-dev-bot) — but see the temporary-state note below | | `remote-dev-bot-tester` | Simulates an unauthorized external user. NOT a collaborator on any repo. | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` (on remote-dev-bot) | -### RDB_TESTER_PAT_TOKEN details - -In practice a PAT on the `gnovak` account (the dedicated `remote-dev-bot` -account was flagged by GitHub as a bot, so authorized-trigger duty moved to -the owner account; test activity therefore does show up in `gnovak`'s -contribution stats). - -- Recommended: fine-grained PAT scoped to `gnovak/remote-dev-bot-test` only, - with Contents read/write, Issues read/write, Pull requests read/write, and - Actions read/write (the suite monitors and cancels workflow runs). - A classic PAT with `repo` scope also works but grants far more than needed. -- Prefer a long or no expiration, and note the date somewhere visible — an - expired token surfaces as the e2e pre-flight failing with 401 (the - 2026-08-05 full-suite failure was exactly this, from a 90-day default). -- The token owner must pass the security gate on `remote-dev-bot-test` - (owner or collaborator) so its trigger comments start runs. +### remote-dev-bot (bot account) details + +The intended setup: + +- Classic PAT with `public_repo` + `workflow` scopes, no expiration +- **`public_repo`** — create issues, post comments, open PRs in public repos + (standard e2e test flow) +- **`workflow`** — read/write `.github/workflows/` files; required when the + agent itself needs to modify workflow files in a PR (e.g., the dogfood use + case where rdb is editing its own workflow files). Without this scope, + writes to `.github/workflows/` fail with HTTP 404. +- Must be a collaborator on `remote-dev-bot-test` so the security gate allows + its trigger comments +- Keeps test activity out of `gnovak`'s GitHub contribution stats + +> **Temporary state (as of 2026-08):** the `remote-dev-bot` account is +> flagged by GitHub (its profile 404s on the public API), so tokens cannot +> be minted on it until that's resolved with GitHub support. Until then, +> `RDB_TESTER_PAT_TOKEN` holds a PAT minted on `gnovak` instead (verifiable: +> e2e test issues on rdb-test are authored by `gnovak`). If minting the +> stopgap on `gnovak`, prefer a fine-grained PAT scoped to +> `gnovak/remote-dev-bot-test` only (Contents/Issues/Pull requests +> read-write + Actions read-write) with a long expiry — the 2026-08-05 +> full-suite failure was a 90-day stopgap token quietly expiring. Once the +> bot account is restored, move the token back per the intended setup above. ### remote-dev-bot-tester details @@ -85,7 +94,7 @@ Secrets stored on `gnovak/remote-dev-bot`: | `OPENAI_API_KEY` | OpenAI API key (for GPT models) | | `GEMINI_API_KEY` | Google AI API key (for Gemini models) | | `RDB_APP_PRIVATE_KEY` | GitHub App private key, for bot identity on comments/PRs. | -| `RDB_TESTER_PAT_TOKEN` | PAT for an account authorized on rdb-test (in practice `gnovak` — see account table). Used by e2e tests to create issues and post authorized trigger comments. | +| `RDB_TESTER_PAT_TOKEN` | PAT for `remote-dev-bot` account (collaborator on rdb-test; temporarily a `gnovak` PAT — see account details). Used by e2e tests to post authorized trigger comments. | | `RDB_TESTER_UNAUTHORIZED_PAT_TOKEN` | PAT for `remote-dev-bot-tester` (not a collaborator). Used by security e2e tests to verify unauthorized users are blocked. | Variables stored on `gnovak/remote-dev-bot`: diff --git a/tests/e2e-security.sh b/tests/e2e-security.sh index ab6c03d..f7e3ca5 100755 --- a/tests/e2e-security.sh +++ b/tests/e2e-security.sh @@ -87,7 +87,21 @@ gh_login=$(gh api user --jq .login 2>/dev/null) || { echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 exit 1 } -echo "==> Authenticated to GitHub as: ${gh_login}" +gh_exp=$(gh api user -i 2>/dev/null | grep -i "^github-authentication-token-expiration:" | cut -d" " -f2- | tr -d "\r") +echo "==> Authenticated to GitHub as: ${gh_login}${gh_exp:+ (token expires: ${gh_exp})}" + +# Validate the unauthorized-user token too, when present — its failures +# otherwise surface only mid-gating-test. +if [[ -n "${UNAUTHORIZED_PAT:-}" ]]; then + ua_login=$(curl -fs -H "Authorization: token ${UNAUTHORIZED_PAT}" \ + https://api.github.com/user 2>/dev/null | python3 -c "import json,sys; print(json.load(sys.stdin)['login'])" 2>/dev/null) || true + if [[ -z "${ua_login:-}" ]]; then + echo "ERROR: UNAUTHORIZED_PAT is invalid or expired (api.github.com/user failed)." >&2 + echo "ERROR: In CI this is secrets.RDB_TESTER_UNAUTHORIZED_PAT_TOKEN (remote-dev-bot-tester account)." >&2 + exit 1 + fi + echo "==> Unauthorized-user token authenticates as: ${ua_login}" +fi check_graphql_quota diff --git a/tests/e2e.sh b/tests/e2e.sh index 250dabc..2c2b255 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -83,7 +83,8 @@ gh_login=$(gh api user --jq .login 2>/dev/null) || { echo "ERROR: mint a new PAT with write access to the test repo and update the secret." >&2 exit 1 } -echo "==> Authenticated to GitHub as: ${gh_login}" +gh_exp=$(gh api user -i 2>/dev/null | grep -i "^github-authentication-token-expiration:" | cut -d" " -f2- | tr -d "\r") +echo "==> Authenticated to GitHub as: ${gh_login}${gh_exp:+ (token expires: ${gh_exp})}" check_graphql_quota From fe5b877196034d040a2356737b9285734d9b0881 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sat, 8 Aug 2026 05:46:31 +0000 Subject: [PATCH 33/39] gpt-medium/gpt-large: co-point at gpt-5.3-codex (Sol can't run the tool loop) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-08-08 full-suite run showed openai/gpt-5.6-sol rejecting function tools on chat completions ("use /v1/responses or set reasoning_effort to 'none'"), failing both aliases at iteration 1. Per the alias rules — capability at agentic coding is the admission test and every alias must resolve to a runnable model — the OpenAI ladder compresses fully onto the Codex line until a Responses API path exists (tracked in issue 653). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- remote-dev-bot.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/remote-dev-bot.yaml b/remote-dev-bot.yaml index 51b1e96..318bb52 100644 --- a/remote-dev-bot.yaml +++ b/remote-dev-bot.yaml @@ -171,12 +171,12 @@ models: description: "OpenAI agentic-coding workhorse (Codex line)" gpt-medium: - id: openai/gpt-5.6-sol - description: "OpenAI flagship" + id: openai/gpt-5.3-codex + description: "Co-pointed at gpt-small — gpt-5.6-sol requires the Responses API for tool use (issue #653)" gpt-large: - id: openai/gpt-5.6-sol - description: "Co-pointed at gpt-medium — no distinct OpenAI frontier tier suits agentic coding today" + id: openai/gpt-5.3-codex + description: "Co-pointed at gpt-small — no OpenAI model above the Codex line runs rdb's tool loop today" gemini-small: id: gemini/gemini-3.6-flash From 16e1d5402ebacbd9ce5f3fd695c77890605852f6 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sat, 8 Aug 2026 06:02:32 +0000 Subject: [PATCH 34/39] dogfood.yml: remove invalid `workflows: write` permissions key GitHub Actions has no `workflows` permissions scope; validation began hard-rejecting it around 2026-05-01 ("Invalid workflow file ... Line 26: Unexpected value 'workflows'"). Since then every push to any branch has spawned a phantom failed dogfood run (100+ red Xs since May), and real /dogfood comment triggers fail the same validation. Workflow-file write access comes from the GitHub App's Workflows permission, not a job permissions key, so nothing is lost by removing it. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- .github/workflows/dogfood.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dogfood.yml b/.github/workflows/dogfood.yml index 6bea054..3847d4f 100644 --- a/.github/workflows/dogfood.yml +++ b/.github/workflows/dogfood.yml @@ -19,11 +19,15 @@ on: pull_request_review_comment: types: [created] +# NOTE: `workflows: write` is not a valid permissions scope — GitHub began +# hard-rejecting it ~2026-05-01 ("Unexpected value 'workflows'"), which made +# every push spawn a phantom failed dogfood run and broke /dogfood comments. +# Workflow-file write access is granted by the App's Workflows permission +# (see CONTRIBUTING.md), not by a job permissions key. permissions: contents: write issues: write pull-requests: write - workflows: write jobs: resolve: From 65fabf84e92d359f007e92752094aa3cbe498f8a Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sat, 8 Aug 2026 06:03:53 +0000 Subject: [PATCH 35/39] test_yaml: invert the workflows-permission regression test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test_dogfood_has_workflows_write_permission (from issue #463) asserted the presence of the exact key that now invalidates the workflow file. The key never did anything — GITHUB_TOKEN cannot modify workflow files regardless of job permissions; that capability comes from the GitHub App's Workflows permission. Replaced with the inverse guard across all workflow files. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- tests/test_yaml.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/tests/test_yaml.py b/tests/test_yaml.py index 166b632..b64dc4d 100644 --- a/tests/test_yaml.py +++ b/tests/test_yaml.py @@ -202,17 +202,24 @@ def find_target_checkout(job): ) -def test_dogfood_has_workflows_write_permission(): - """dogfood.yml must include workflows:write so the fallback github.token - can push workflow file changes when no app token is configured. - - Regression test for issue #463. +def test_no_invalid_workflows_permission_key(): + """No workflow may use `workflows:` in a permissions block — it is not a + valid GitHub Actions permissions scope, and since ~2026-05-01 GitHub + hard-rejects it ("Unexpected value 'workflows'"), invalidating the whole + workflow file. It was originally added for issue #463 in the belief it + let github.token push workflow files; GITHUB_TOKEN can never modify + workflow files regardless of permissions — that capability comes from + the GitHub App's Workflows permission (see CONTRIBUTING.md). + + Regression test for the May-August 2026 phantom dogfood failures. """ - dogfood = load_yaml(REPO_ROOT / ".github/workflows/dogfood.yml") - permissions = dogfood.get("permissions", {}) - assert permissions.get("workflows") == "write", ( - "dogfood.yml must have 'workflows: write' in permissions" - ) + for wf in (REPO_ROOT / ".github/workflows").glob("*.yml"): + config = load_yaml(wf) + permissions = config.get("permissions", {}) or {} + assert "workflows" not in permissions, ( + f"{wf.name}: 'workflows' is not a valid permissions scope and " + f"invalidates the workflow file" + ) # --- Security checks --- From 8533e725d9286b8155b443dc478f8f227f702846 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Sat, 8 Aug 2026 06:39:57 +0000 Subject: [PATCH 36/39] Repoint gemini aliases to 2.5-flash; harden timeout test against fast models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes from the 2026-08-08 full-suite runs: 1. gemini-3.6-flash fails rdb's loop 4-of-6 times today on the litellm thought-signature converter bug (issue 654; upstream litellm 16893 / 25322 family — no fixed release as of 1.95.0/1.96.0rc1). Per the alias admission test, the gemini trio repoints to gemini-2.5-flash — the best Gemini that actually runs the loop (it predates thought signatures entirely and passed every May e2e window). Restore 3.6-flash when litellm's handling stabilizes. 2. The timeout test was outrun a second time: a model scripted the 50-file task in one bash call inside the 30s watchdog window. Third hardening makes wall-clock the constraint instead of reasoning speed: 200 files in 200 individual git commits (~40-60s of git execution alone), which no model can compress below the watchdog regardless of how fast it thinks. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- remote-dev-bot.yaml | 10 +++++----- tests/e2e.sh | 16 ++++++++++------ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/remote-dev-bot.yaml b/remote-dev-bot.yaml index 318bb52..3edb00a 100644 --- a/remote-dev-bot.yaml +++ b/remote-dev-bot.yaml @@ -179,15 +179,15 @@ models: description: "Co-pointed at gpt-small — no OpenAI model above the Codex line runs rdb's tool loop today" gemini-small: - id: gemini/gemini-3.6-flash - description: "Google workhorse — strongest Gemini for agentic coding today" + id: gemini/gemini-2.5-flash + description: "Google workhorse — best Gemini that runs rdb's loop today (3.6-flash blocked on litellm thought-signature bug, issue #654)" gemini-medium: - id: gemini/gemini-3.6-flash - description: "Co-pointed at gemini-small — repoint to Gemini 3.5 Pro if/when it reaches GA" + id: gemini/gemini-2.5-flash + description: "Co-pointed at gemini-small — repoint when a stronger Gemini clears the tool-loop admission test" gemini-large: - id: gemini/gemini-3.6-flash + id: gemini/gemini-2.5-flash description: "Co-pointed — no distinct Google frontier tier today" # Agent settings — apply to all modes unless overridden per-mode above. diff --git a/tests/e2e.sh b/tests/e2e.sh index 2c2b255..f911b35 100755 --- a/tests/e2e.sh +++ b/tests/e2e.sh @@ -393,23 +393,27 @@ post_issue_comment "$RF_ISSUE_NUM" "$TEST_REPO" "/agent-resolve" # --- Trigger timeout test --- # Purpose: verify the watchdog kills the agent process when it exceeds # timeout_minutes. The task must be concrete enough that the agent cannot -# scope-reduce its way to an early finish() — and the timeout short enough -# that even fast scope-reduced work doesn't fit. (The previous vague -# "refactor everything" prompt with timeout_minutes=1 let the agent finish -# in 50s by creating one file with type hints; watchdog never fired.) +# scope-reduce its way to an early finish() — and slow enough in WALL-CLOCK +# terms that no model can outrun the watchdog. Two prior hardenings: +# (1) the vague "refactor everything" prompt with timeout_minutes=1 let the +# agent finish in 50s by creating one file; (2) the 50-files-one-PR version +# was outrun on 2026-08-08 by a model that scripted all 50 files in a +# single bash call inside the 30s window. The current design demands 200 +# individual git commits — tool-execution wall time (~40-60s of git alone) +# exceeds the watchdog no matter how fast the model reasons. log "Creating timeout test issue..." timeout_ts=$(date +%s) timeout_title="Test: timeout enforcement (e2e-timeout-$timeout_ts)" timeout_issue_url=$(gh issue create --repo "$TEST_REPO" \ --title "$timeout_title" \ - --body "Create exactly 50 Python files named day_01.py through day_50.py in a new \`days/\` directory at the repo root. Each file must contain exactly one function with this exact signature and body (substitute NN with the day number, NOT zero-padded in the body): + --body "Create exactly 200 Python files named day_001.py through day_200.py in a new \`days/\` directory at the repo root. Each file must contain exactly one function with this exact signature and body (substitute NN with the day number, NOT zero-padded in the body): \`\`\`python def visits_on_day_NN() -> int: return NN \`\`\` -All 50 files must be created and committed in a single PR. This is a concrete, fully-specified task that cannot be scope-reduced — produce all 50 files or report failure.") +IMPORTANT: each file must be added in its own individual git commit — 200 separate commits, one file per commit, with the commit message 'Add day NNN'. Do not batch files into shared commits. This is a concrete, fully-specified task that cannot be scope-reduced — produce all 200 files in 200 commits or report failure.") timeout_issue_num="${timeout_issue_url##*/}" cleanup_issues+=("$timeout_issue_num") From 4ae130dfb1d0cd9b489c01459857fb4e7306642d Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Mon, 10 Aug 2026 03:57:56 +0000 Subject: [PATCH 37/39] Draft v1.0.0 changelog Comprehensive AI draft covering v0.9.0..dev (31 merges): the alias tier system, zero-config install, the review-driven security and reliability campaign, cost/caching correctness, delegate fixes, test infrastructure, and known issues. Awaiting the mandatory human pare-down before tagging. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- CHANGELOG.md | 143 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c1757d..beb7be9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,148 @@ # Changelog +## v1.0.0 — Stabilization, model tiers, zero-config install (Aug 2026) + +The 1.0 release. Everything since v0.9.0 was driven by a comprehensive +code/workflow/docs/usage review (checked into the repo under +`comprehensive-review/2026-06-11/`) followed by a fix campaign, plus a +redesign of the model alias system for long-term stability. The release +gate was a fully green run of the complete test suite — unit tests, e2e +across every model alias on all three providers, and the security suite. + +### Model aliases: three capability tiers + +- **New tier system**: aliases are now `{provider}-{small,medium,large}` — + small = best value agentic-coding workhorse, medium = the provider's + flagship, large = frontier-above-flagship where one exists. Tiers are + anchored on capability at agentic coding (price is an expectation, not + the admission test), the ladder is monotone, and every alias always + resolves to a runnable model — providers lacking a distinct model for a + tier co-point at the tier below. The reasoning and repointing rules are + documented in `remote-dev-bot.yaml` and the README. +- **Breaking (pre-1.0)**: `claude-large` now means the frontier tier + (Claude Fable 5); the Opus flagship moved to `claude-medium`. `gpt-*` + and `gemini-*` gained `medium` tiers. +- **Current mappings**: claude → sonnet-5 / opus-5 / fable-5; gpt → all + tiers on gpt-5.3-codex (GPT-5.6 Sol cannot run tool loops via chat + completions — restore tracked in #653); gemini → all tiers on + 2.5-flash (gemini-3.6-flash blocked on a litellm thought-signature + bug — #654). +- **Post-1.0 policy**: the alias set is append-only; repointing an alias + is expected and gets a changelog entry. Pin your own aliases in your + repo's `models:` section if you'd draw the lines differently. + +### Zero-config install + +- **`default_model: auto`**: fresh installs no longer fail when no model + is configured. The default resolves from whichever provider API key + secrets are present (priority: `ANTHROPIC_API_KEY` > `OPENAI_API_KEY` > + `GEMINI_API_KEY`). Explicit configuration always wins; no keys at all + fails at parse time with an actionable message. +- **install.md rewritten around optional config**: the per-repo + `remote-dev-bot.yaml` is optional; the previous instructions curl'd a + template file that had been deleted (every fresh install 404'd at that + step). The base config in this repo is now the live reference. + +### Security hardening + +- **PR-title shell injection closed**: `create_pr` built a `shell=True` + command with the LLM-authored title escaped only for double quotes; + backticks and `$(...)` from a prompt-injected issue could execute on + the runner. Now an argv-list subprocess call with no shell. +- **Config values no longer interpolated into workflow Python**: + `extra_instructions` and friends were spliced into heredoc Python + source in 7 workflow steps — multi-line values (the documented format) + crashed the step, and target-repo config could inject code. All sites + now pass values through step `env:` blocks. +- **Phantom security docs removed**: the README described a + `SECURITY_GATE` marker and "security microagent" that did not exist; + the real gate (author association) is now documented as such. + +### Reliability fixes (from the 2026-06-11 review) + +- **Context-overflow recovery un-deadened**: `ContextWindowExceededError` + subclasses `BadRequestError` and was caught by the wrong handler, so + the emergency trim + graceful wrap-up path never ran. Handler order + fixed; overflowing runs now wrap up instead of dying. +- **Compaction no longer corrupts tool-call pairing**: the boundary cut + could orphan tool results from their tool calls, producing message + lists providers reject with a 400. The cut now respects group + boundaries. +- **resolve.py uses the shared read/write-aware context trimming** (its + stale local copy dropped write results as readily as reads). +- **Reconcile hardened**: transient provider errors (529s) are retried + like the sibling loops; context/truncation config is actually plumbed + into the job; the rebase conflict-marker explanation in the prompt had + the sides reversed and is now correct. +- **Silent-failure statuses**: the "no tool calls ×3" break paths in + resolve and reconcile now write failure statuses instead of ending + runs with no explanation. +- **CI on dev PRs**: unit tests now run on PRs targeting `dev` (the + development branch model) — previously only `main`, so dev PRs merged + with zero checks. +- **/dogfood repaired**: an invalid `workflows: write` permissions key + (hard-rejected by GitHub since ~May) invalidated the workflow file, + breaking `/dogfood` comments and attaching a phantom failed run to + every push for three months. + +### Cost & caching correctness + +- **Design loops now use prompt caching**: the moving-tail cache marker + (in resolve and reconcile since v0.7) was ported to the design loop — + standalone `/agent-design` and workshop/delegate design stages were + paying full input price every iteration on Anthropic models. +- **Cache-write tokens actually tracked**: all three loops read a + litellm field that doesn't exist (`cache_creation_input_tokens` vs + `cache_creation_tokens`), so cache-write costs never appeared and + savings were overstated. +- **Design-mode distillation un-broken**: a 5-vs-6 tuple unpack mismatch + silently disabled the distillation pre-pass for every `/agent-design` + run (the swallowed error meant full-repo context every time). +- **Distillation and workshop/delegate costs fully accounted**: + structural-extract signatures no longer degrade to bare argument + names, and distillation cost is included in workshop/delegate totals. + +### Delegate & loop behavior + +- **Delegate Stage 3 safety blocking fixed**: a revised design containing + `/agent` commands is now discarded (falling back to the vetted Stage 1 + design) instead of merely not being posted while still flowing to + later stages. +- **Design-stage wrapup fixed**: the wrap-up nudge iteration was computed + against the code budget (40 of 50) and passed into 15-iteration design + loops, where it could never fire. Now rescaled per stage. +- **Wrapup mechanics unified**: all three loops re-inject the wrap-up + nudge every iteration past the threshold (deliberate escalation — + previously three different implementations). +- **Configured design/review iteration budgets respected**: the parse + job never exported `design_max_iterations` / `review_max_iterations`, + so design loops ran hardcoded defaults and inline overrides were + silently ignored. + +### Test infrastructure + +- **e2e pre-flight**: both e2e scripts validate GitHub tokens up front + and print the token owner and expiry, failing fast with rotation + instructions instead of mid-suite 401 confusion. +- **Reconcile e2e failures now count** toward the suite exit code + (previously a reconcile failure could exit green). +- **Timeout test hardened (third design)**: the watchdog test now + demands 200 individual git commits, making wall-clock the constraint — + fast models had twice outrun task designs that raced reasoning speed. + +### Known issues + +- `gemini-3.6-flash` is temporarily off the alias table: litellm's + Gemini 3.x thought-signature handling intermittently breaks multi-turn + tool calling (#654; upstream litellm #16893/#25322). The gemini tiers + point at `gemini-2.5-flash` until a fixed litellm release lands. +- GPT-5.6 Sol requires OpenAI's Responses API for tool use; the gpt + tiers point at `gpt-5.3-codex` until litellm's responses bridge covers + it (#653). + +**Breaking changes:** model alias renames described above (pre-1.0, no +compatibility guarantee was in effect). Post-1.0, alias names are stable. + ## v0.9.0 — Delegate mode, reconcile mode, council reviews (May 2026) - **Delegate mode** (`/agent-delegate`): Full design-to-implementation pipeline From 78ce625a06ca1972a17f977bf46ac7a312b7b3af Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Wed, 19 Aug 2026 17:00:22 +0000 Subject: [PATCH 38/39] Pare down v1.0.0 changelog per review Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --- CHANGELOG.md | 168 +++++++++------------------------------------------ 1 file changed, 30 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index beb7be9..471e15a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,147 +1,39 @@ # Changelog -## v1.0.0 — Stabilization, model tiers, zero-config install (Aug 2026) - -The 1.0 release. Everything since v0.9.0 was driven by a comprehensive -code/workflow/docs/usage review (checked into the repo under -`comprehensive-review/2026-06-11/`) followed by a fix campaign, plus a -redesign of the model alias system for long-term stability. The release -gate was a fully green run of the complete test suite — unit tests, e2e -across every model alias on all three providers, and the security suite. - -### Model aliases: three capability tiers - -- **New tier system**: aliases are now `{provider}-{small,medium,large}` — - small = best value agentic-coding workhorse, medium = the provider's - flagship, large = frontier-above-flagship where one exists. Tiers are - anchored on capability at agentic coding (price is an expectation, not - the admission test), the ladder is monotone, and every alias always - resolves to a runnable model — providers lacking a distinct model for a - tier co-point at the tier below. The reasoning and repointing rules are - documented in `remote-dev-bot.yaml` and the README. -- **Breaking (pre-1.0)**: `claude-large` now means the frontier tier - (Claude Fable 5); the Opus flagship moved to `claude-medium`. `gpt-*` - and `gemini-*` gained `medium` tiers. -- **Current mappings**: claude → sonnet-5 / opus-5 / fable-5; gpt → all - tiers on gpt-5.3-codex (GPT-5.6 Sol cannot run tool loops via chat - completions — restore tracked in #653); gemini → all tiers on - 2.5-flash (gemini-3.6-flash blocked on a litellm thought-signature - bug — #654). -- **Post-1.0 policy**: the alias set is append-only; repointing an alias - is expected and gets a changelog entry. Pin your own aliases in your - repo's `models:` section if you'd draw the lines differently. - -### Zero-config install - -- **`default_model: auto`**: fresh installs no longer fail when no model - is configured. The default resolves from whichever provider API key - secrets are present (priority: `ANTHROPIC_API_KEY` > `OPENAI_API_KEY` > - `GEMINI_API_KEY`). Explicit configuration always wins; no keys at all - fails at parse time with an actionable message. -- **install.md rewritten around optional config**: the per-repo - `remote-dev-bot.yaml` is optional; the previous instructions curl'd a - template file that had been deleted (every fresh install 404'd at that - step). The base config in this repo is now the live reference. - -### Security hardening - -- **PR-title shell injection closed**: `create_pr` built a `shell=True` - command with the LLM-authored title escaped only for double quotes; - backticks and `$(...)` from a prompt-injected issue could execute on - the runner. Now an argv-list subprocess call with no shell. -- **Config values no longer interpolated into workflow Python**: - `extra_instructions` and friends were spliced into heredoc Python - source in 7 workflow steps — multi-line values (the documented format) - crashed the step, and target-repo config could inject code. All sites - now pass values through step `env:` blocks. -- **Phantom security docs removed**: the README described a - `SECURITY_GATE` marker and "security microagent" that did not exist; - the real gate (author association) is now documented as such. - -### Reliability fixes (from the 2026-06-11 review) - -- **Context-overflow recovery un-deadened**: `ContextWindowExceededError` - subclasses `BadRequestError` and was caught by the wrong handler, so - the emergency trim + graceful wrap-up path never ran. Handler order - fixed; overflowing runs now wrap up instead of dying. -- **Compaction no longer corrupts tool-call pairing**: the boundary cut - could orphan tool results from their tool calls, producing message - lists providers reject with a 400. The cut now respects group - boundaries. -- **resolve.py uses the shared read/write-aware context trimming** (its - stale local copy dropped write results as readily as reads). -- **Reconcile hardened**: transient provider errors (529s) are retried - like the sibling loops; context/truncation config is actually plumbed - into the job; the rebase conflict-marker explanation in the prompt had - the sides reversed and is now correct. -- **Silent-failure statuses**: the "no tool calls ×3" break paths in - resolve and reconcile now write failure statuses instead of ending - runs with no explanation. -- **CI on dev PRs**: unit tests now run on PRs targeting `dev` (the - development branch model) — previously only `main`, so dev PRs merged - with zero checks. -- **/dogfood repaired**: an invalid `workflows: write` permissions key - (hard-rejected by GitHub since ~May) invalidated the workflow file, - breaking `/dogfood` comments and attaching a phantom failed run to - every push for three months. - -### Cost & caching correctness - -- **Design loops now use prompt caching**: the moving-tail cache marker - (in resolve and reconcile since v0.7) was ported to the design loop — - standalone `/agent-design` and workshop/delegate design stages were - paying full input price every iteration on Anthropic models. -- **Cache-write tokens actually tracked**: all three loops read a - litellm field that doesn't exist (`cache_creation_input_tokens` vs - `cache_creation_tokens`), so cache-write costs never appeared and - savings were overstated. -- **Design-mode distillation un-broken**: a 5-vs-6 tuple unpack mismatch - silently disabled the distillation pre-pass for every `/agent-design` - run (the swallowed error meant full-repo context every time). -- **Distillation and workshop/delegate costs fully accounted**: - structural-extract signatures no longer degrade to bare argument - names, and distillation cost is included in workshop/delegate totals. - -### Delegate & loop behavior - -- **Delegate Stage 3 safety blocking fixed**: a revised design containing - `/agent` commands is now discarded (falling back to the vetted Stage 1 - design) instead of merely not being posted while still flowing to - later stages. -- **Design-stage wrapup fixed**: the wrap-up nudge iteration was computed - against the code budget (40 of 50) and passed into 15-iteration design - loops, where it could never fire. Now rescaled per stage. -- **Wrapup mechanics unified**: all three loops re-inject the wrap-up - nudge every iteration past the threshold (deliberate escalation — - previously three different implementations). -- **Configured design/review iteration budgets respected**: the parse - job never exported `design_max_iterations` / `review_max_iterations`, - so design loops ran hardcoded defaults and inline overrides were - silently ignored. - -### Test infrastructure - -- **e2e pre-flight**: both e2e scripts validate GitHub tokens up front - and print the token owner and expiry, failing fast with rotation - instructions instead of mid-suite 401 confusion. -- **Reconcile e2e failures now count** toward the suite exit code - (previously a reconcile failure could exit green). -- **Timeout test hardened (third design)**: the watchdog test now - demands 200 individual git commits, making wall-clock the constraint — - fast models had twice outrun task designs that raced reasoning speed. +## v1.0.0 (Aug 2026) + +- **Model aliases: three capability tiers.** Aliases are now + `{provider}-{small,medium,large}`: small = best value-for-money + agentic-coding workhorse, medium = the provider's flagship, large = + frontier-above-flagship where one exists. Tiers are anchored on + capability at agentic coding (price is an expectation, not the + admission test) and the ladder is monotone — going up a tier never + gets a weaker model. The model landscape doesn't always map cleanly + onto three tiers, so aliases sometimes point at the same model rather + than leaving a tier undefined or pointing it at something that can't + do the job. Repointing rules and reasoning are documented in + `remote-dev-bot.yaml` and the README. +- **Zero-config default model** (`default_model: auto`): fresh installs + run without configuring a model at all — the default resolves from + whichever provider API key secret is present. Explicit configuration + always wins. +- **Lots of small fixes and cleanup in preparation for 1.0**: security + hardening, reliability and prompt-cache correctness, cost-accounting + accuracy, doc repair, and test-infrastructure work, driven by a + comprehensive pre-1.0 review (`comprehensive-review/2026-06-11/`). ### Known issues -- `gemini-3.6-flash` is temporarily off the alias table: litellm's - Gemini 3.x thought-signature handling intermittently breaks multi-turn - tool calling (#654; upstream litellm #16893/#25322). The gemini tiers - point at `gemini-2.5-flash` until a fixed litellm release lands. -- GPT-5.6 Sol requires OpenAI's Responses API for tool use; the gpt - tiers point at `gpt-5.3-codex` until litellm's responses bridge covers - it (#653). +- The gemini tiers point at `gemini-2.5-flash` rather than the newer + 3.x models: litellm's Gemini 3.x thought-signature handling + intermittently breaks multi-turn tool calling (#654). +- The gpt tiers point at `gpt-5.3-codex` rather than GPT-5.6 Sol: the + 5.6 frontier family requires OpenAI's Responses API for tool use, + which rdb's litellm path doesn't use yet (#653). -**Breaking changes:** model alias renames described above (pre-1.0, no -compatibility guarantee was in effect). Post-1.0, alias names are stable. +**Breaking changes:** model aliases renamed as part of the tier system +(pre-1.0, no compatibility guarantee was in effect). Post-1.0, alias +names are stable and the alias set is append-only. ## v0.9.0 — Delegate mode, reconcile mode, council reviews (May 2026) From 26522533f7ed7348126e1a603726700f8f6494a8 Mon Sep 17 00:00:00 2001 From: Greg Novak <greg.novak@gmail.com> Date: Mon, 15 Jun 2026 04:10:22 +0000 Subject: [PATCH 39/39] agents: make CLAUDE.md / GEMINI.md real symlinks to AGENTS.md These were pointer files whose body just said "read AGENTS.md". That indirection has a failure mode: an agent can stop at the pointer and miss edits to AGENTS.md. A real symlink makes CLAUDE.md / GEMINI.md *be* AGENTS.md at the filesystem level, so the canonical content is always what's read. Git stores symlinks natively (mode 120000), so this travels correctly on clone. CLAUDE.md -> AGENTS.md; .gemini/GEMINI.md -> ../AGENTS.md. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com> --- .gemini/GEMINI.md | 4 +--- CLAUDE.md | 7 +------ 2 files changed, 2 insertions(+), 9 deletions(-) mode change 100644 => 120000 .gemini/GEMINI.md mode change 100644 => 120000 CLAUDE.md diff --git a/.gemini/GEMINI.md b/.gemini/GEMINI.md deleted file mode 100644 index 131cbc0..0000000 --- a/.gemini/GEMINI.md +++ /dev/null @@ -1,3 +0,0 @@ -# GEMINI.md - -**Read AGENTS.md in the repository root now before doing anything else.** It contains project conventions, key files, PR policy, and code style guidelines that govern all work in this repo. diff --git a/.gemini/GEMINI.md b/.gemini/GEMINI.md new file mode 120000 index 0000000..be77ac8 --- /dev/null +++ b/.gemini/GEMINI.md @@ -0,0 +1 @@ +../AGENTS.md \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 54fe34a..0000000 --- a/CLAUDE.md +++ /dev/null @@ -1,6 +0,0 @@ -# CLAUDE.md - -**Read [AGENTS.md](AGENTS.md) now before responding.** It contains project -conventions that govern all work in this repo. - -Global personal preferences are in `~/.config/agents/AGENTS.md`. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file