From 2cc4ca8293e71ac21885336b4f252b9d7e67bcb8 Mon Sep 17 00:00:00 2001 From: Nathanael Roberts Date: Sat, 5 Sep 2026 03:22:15 -0700 Subject: [PATCH 1/4] feat(watch): shadow graded wedge score and settlement records Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- bin/argyle_gates.py | 241 +++++++++++++++++++++++++++++++++++ bin/fm-test-run.sh | 1 + bin/fm-watch.sh | 45 +++++++ bin/fm-wedge-score.py | 151 ++++++++++++++++++++++ tests/fm-wedge-score.test.sh | 106 +++++++++++++++ 5 files changed, 544 insertions(+) create mode 100644 bin/argyle_gates.py create mode 100755 bin/fm-wedge-score.py create mode 100755 tests/fm-wedge-score.test.sh diff --git a/bin/argyle_gates.py b/bin/argyle_gates.py new file mode 100644 index 00000000000..711308bf57d --- /dev/null +++ b/bin/argyle_gates.py @@ -0,0 +1,241 @@ +# Provenance: lifted verbatim from cipherholdingsllc/nakatomi +# branch t2-argyle-frontier, path crypto-crawler/crypto_crawler/argyle/gates.py +# (commit 02be13ca139e847089d3ce7c6503d9f5dae914ba). +# Pure stdlib. Do not edit here; upstream fixes land in nakatomi first. +""" +Promotion gates — the bar every learned knob must clear before it may touch +the mock book. Three tests, ALL required (logical AND): + + 1. Deflated Sharpe (Bailey & López de Prado 2014): the observed Sharpe must + be significant AFTER deflating for the number of configurations actually + tried. Sweeping N configs and reporting the best is the quant original + sin; the deflation term grows with N, so a small sample cannot launder a + lucky config into a promotion. + 2. Block-permutation test: the strategy's edge statistic must beat p < 0.05 + against block-shuffled nulls (blocks preserve short-range autocorrelation + that a naive i.i.d. shuffle would destroy — shuffling away autocorrelation + makes nulls too easy to beat). + 3. Out-of-sample calibration: mean Brier of the candidate's p_up on held-out + outcomes must beat CLIMATOLOGY — the constant base-rate predictor, whose + Brier is p̄(1−p̄) ≤ 0.25. A fixed 0.25 bar is launderable at skewed base + rates (constant p=0.9 on 90%-up outcomes scores 0.09 with zero skill — + found by adversarial audit 2026-07-04); climatology is not. + +Everything is pure Python + math (no scipy — this box is minimal, and a gate +must not fail-open because a dependency is missing). Normal CDF via math.erf; +inverse normal via Acklam's rational approximation (|ε| < 1.15e-9, far below +any decision boundary here). + +Honest-refusal posture: with too little data the gate returns passed=False +with reason "insufficient data" — it never extrapolates a pass. On ~teens of +settled findings the EXPECTED outcome is failure; that is the gate working. +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field + +EULER_GAMMA = 0.5772156649015329 + + +def norm_cdf(x: float) -> float: + return 0.5 * (1.0 + math.erf(x / math.sqrt(2.0))) + + +def norm_ppf(p: float) -> float: + """Acklam's inverse-normal approximation. Pure, dependency-free.""" + if not 0.0 < p < 1.0: + raise ValueError(f"p must be in (0,1), got {p}") + a = (-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, + 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00) + b = (-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, + 6.680131188771972e+01, -1.328068155288572e+01) + c = (-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, + -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00) + d = (7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, + 3.754408661907416e+00) + p_low, p_high = 0.02425, 1 - 0.02425 + if p < p_low: + q = math.sqrt(-2 * math.log(p)) + return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / \ + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + if p > p_high: + q = math.sqrt(-2 * math.log(1 - p)) + return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / \ + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + q = p - 0.5 + r = q * q + return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / \ + (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1) + + +def _moments(xs: list[float]) -> tuple[float, float, float, float]: + """(mean, std, skew, kurtosis) — kurtosis is NON-excess (normal = 3.0), + matching the PSR formula's convention. Sample std (ddof=1).""" + n = len(xs) + mean = sum(xs) / n + var = sum((x - mean) ** 2 for x in xs) / (n - 1) + std = math.sqrt(var) + if std == 0.0: + return mean, 0.0, 0.0, 3.0 + skew = sum(((x - mean) / std) ** 3 for x in xs) / n + kurt = sum(((x - mean) / std) ** 4 for x in xs) / n + return mean, std, skew, kurt + + +def sharpe(returns: list[float]) -> float: + """Per-period, non-annualized. Sample std.""" + mean, std, _, _ = _moments(returns) + if std == 0.0: + return 0.0 + return mean / std + + +@dataclass +class GateResult: + passed: bool + reason: str + detail: dict = field(default_factory=dict) + + +def deflated_sharpe( + returns: list[float], + n_trials: int, + var_sr_trials: float | None = None, + confidence: float = 0.95, +) -> GateResult: + """DSR = PSR evaluated at the expected-max-Sharpe benchmark under n_trials. + + SR0 = sqrt(V[SR_n]) * ((1-γ)·Φ⁻¹(1-1/N) + γ·Φ⁻¹(1-1/(N·e))) + DSR = Φ( (SR - SR0)·sqrt(T-1) / sqrt(1 - γ₃·SR + (γ₄-1)/4·SR²) ) + + `var_sr_trials` is the cross-trial variance of the Sharpe estimates from + the actual config sweep — pass it when the sweep recorded per-config SRs. + When unavailable, the estimator's own asymptotic variance is used as a + stand-in (documented compromise: it under-deflates a heterogeneous sweep, + so prefer real trial variance for any promotion decision).""" + t = len(returns) + if t < 10: + return GateResult(False, f"insufficient data: {t} returns < 10") + if n_trials < 1: + return GateResult(False, f"n_trials must be >= 1, got {n_trials}") + sr = sharpe(returns) + _, _, skew, kurt = _moments(returns) + denom = 1.0 - skew * sr + (kurt - 1.0) / 4.0 * sr * sr + if denom <= 0.0: + return GateResult(False, "PSR variance term non-positive (pathological moments)", + {"sr": sr, "skew": skew, "kurt": kurt}) + if var_sr_trials is None: + var_sr_trials = denom / (t - 1) + if n_trials == 1: + sr0 = 0.0 + else: + e = math.e + sr0 = math.sqrt(max(var_sr_trials, 0.0)) * ( + (1 - EULER_GAMMA) * norm_ppf(1 - 1.0 / n_trials) + + EULER_GAMMA * norm_ppf(1 - 1.0 / (n_trials * e)) + ) + dsr = norm_cdf((sr - sr0) * math.sqrt(t - 1) / math.sqrt(denom)) + passed = dsr >= confidence + return GateResult( + passed, + f"DSR={dsr:.4f} {'≥' if passed else '<'} {confidence} (SR={sr:.4f}, SR0={sr0:.4f}, " + f"N={n_trials}, T={t})", + {"dsr": dsr, "sr": sr, "sr0": sr0, "n_trials": n_trials, "t": t}, + ) + + +def block_permutation( + directions: list[float], + returns: list[float], + n_perm: int = 2000, + block: int | None = None, + seed: int = 0, + alpha: float = 0.05, +) -> GateResult: + """H0: directions carry no information about returns. Statistic is the + mean directional capture mean(d_i·r_i). Null draws are a CIRCULAR BLOCK + BOOTSTRAP of the return series (blocks drawn with replacement, order + within blocks preserved) — short-range autocorrelation survives, and the + p-value is approximate rather than an exact permutation p. Seeded — same + inputs, same p. Add-one estimator (Phipson & Smyth) so p is never 0.""" + n = len(returns) + if n != len(directions): + return GateResult(False, "directions/returns length mismatch") + if n < 8: + return GateResult(False, f"insufficient data: {n} pairs < 8") + if block is None: + block = max(1, round(n ** (1.0 / 3.0))) # standard b ~ n^(1/3) rule + obs = sum(d * r for d, r in zip(directions, returns)) / n + rng = random.Random(seed) + n_blocks = math.ceil(n / block) + hits = 0 + for _ in range(n_perm): + starts = [rng.randrange(n) for _ in range(n_blocks)] + permuted: list[float] = [] + for s in starts: + for j in range(block): + if len(permuted) == n: + break + permuted.append(returns[(s + j) % n]) + stat = sum(d * r for d, r in zip(directions, permuted)) / n + if stat >= obs: + hits += 1 + p = (hits + 1) / (n_perm + 1) + passed = p < alpha + return GateResult( + passed, + f"block-permutation p={p:.4f} {'<' if passed else '≥'} {alpha} " + f"(obs={obs:.6f}, block={block}, n_perm={n_perm})", + {"p": p, "obs": obs, "block": block, "n_perm": n_perm}, + ) + + +def oos_calibration(p_up: list[float], outcomes: list[int]) -> GateResult: + """Mean Brier of held-out p_up must beat the climatology Brier p̄(1−p̄) — + the score of constantly predicting the realized base rate. This is the + skill bar a constant predictor cannot pass by construction.""" + if len(p_up) != len(outcomes): + return GateResult(False, "p_up/outcomes length mismatch") + if len(p_up) < 10: + return GateResult(False, f"insufficient data: {len(p_up)} pairs < 10") + brier = sum((p - o) ** 2 for p, o in zip(p_up, outcomes)) / len(p_up) + base_rate = sum(outcomes) / len(outcomes) + climatology = base_rate * (1.0 - base_rate) + passed = brier < climatology + return GateResult( + passed, + f"OOS Brier={brier:.4f} {'<' if passed else '≥'} climatology " + f"{climatology:.4f} (p̄={base_rate:.3f}, n={len(p_up)}, coin-flip bound 0.25)", + {"brier": brier, "n": len(p_up), "climatology": climatology, + "base_rate": base_rate, "coin_flip": 0.25}, + ) + + +def promotion_gate( + returns: list[float], + n_trials: int, + directions: list[float], + p_up: list[float], + outcomes: list[int], + var_sr_trials: float | None = None, + seed: int = 0, +) -> GateResult: + """The full gate: deflated Sharpe AND block permutation AND OOS calibration. + Returns one GateResult whose detail carries all three components — this + dict IS the promotion receipt body.""" + dsr = deflated_sharpe(returns, n_trials, var_sr_trials) + perm = block_permutation(directions, returns, seed=seed) + cal = oos_calibration(p_up, outcomes) + passed = dsr.passed and perm.passed and cal.passed + reason = " | ".join( + f"{name}: {'PASS' if g.passed else 'FAIL'} ({g.reason})" + for name, g in (("deflated_sharpe", dsr), ("block_permutation", perm), ("oos_calibration", cal)) + ) + return GateResult(passed, reason, { + "deflated_sharpe": {"passed": dsr.passed, **dsr.detail}, + "block_permutation": {"passed": perm.passed, **perm.detail}, + "oos_calibration": {"passed": cal.passed, **cal.detail}, + }) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 9320178e41e..7a15f463bca 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -141,6 +141,7 @@ family_for_basename() { fm-kimi-harness.test.sh|fm-muse-harness.test.sh|fm-herdr-lab.test.sh|fm-lint.test.sh|\ fm-operational-input.test.sh|fm-pi-primary-types.test.sh|\ fm-send-popup-settle.test.sh|fm-send-settle.test.sh|\ + fm-wedge-score.test.sh|\ fm-subagent-pretool-check.test.sh|\ fm-supervision-instructions.test.sh|fm-task-delivery.test.sh|\ fm-tmux-submit-busy.test.sh|fm-trace-context-lib.test.sh|\ diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 2f150af60d8..e3abf037835 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -267,6 +267,45 @@ recorded_windows() { # below). FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} +# Shadow mode records wedge settlements and logs a graded score beside the fixed +# timer, but never changes escalation. +wedge_shadow_settle() { # + local win=$1 task=$2 idle=$3 outcome=$4 key + [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + key=$(printf '%s' "$win" | tr ':/.' '___') + "$SCRIPT_DIR/fm-wedge-score.py" settle --home "$FM_HOME" --key "$key" \ + --window "$win" --task "$task" --idle-secs "$idle" --outcome "$outcome" \ + >/dev/null 2>&1 || true +} + +wedge_shadow_score() { # + local task=$2 idle=$3 lane=unknown meta score + [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + meta="$STATE/$task.meta" + if [ -n "$task" ] && [ -f "$meta" ]; then + lane=$(grep '^project=' "$meta" | cut -d= -f2- || true) + [ -n "$lane" ] || lane=unknown + fi + score=$("$SCRIPT_DIR/fm-wedge-score.py" score --home "$FM_HOME" \ + --lane "$lane" --idle-secs "$idle" 2>/dev/null) || true + [ -n "$score" ] && triage_log "shadow wedge score: $score" || true +} + +wedge_shadow_resumed() { # + local win=$1 task=$2 since_file=$3 since idle + [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + [ -e "$since_file" ] || return 0 + since=$(cat "$since_file" 2>/dev/null || true) + case "$since" in + ''|*[!0-9]*) return 0 ;; + esac + idle=$(( $(date +%s) - since )) + wedge_shadow_settle "$win" "$task" "$idle" resumed || true +} + # Repeat-poll wedge-timer bookkeeping for an already-classified stale hash # absorbed as provably-working - repairs a missing/corrupt timer (self-heals a # watcher restart between recording the hash and recording the timer), or @@ -292,6 +331,8 @@ wedge_timer_check() { # key=${key//\//_} key=${key//./_} clear_pause_state "$win" + wedge_shadow_resumed "$win" "$(window_to_task "$win" "$STATE")" \ + "$STATE/.stale-since-$key" || true rm -f "$STATE/.stale-$key" "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key" } @@ -1061,6 +1104,7 @@ EOF if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then wedge_timer_check "$w" "$ssf" "busy (no completed turn)" "$ewf" else + wedge_shadow_resumed "$w" "$task" "$ssf" || true rm -f "$ssf" "$ewf" fi if [ -e "$pf" ] && { [ "$n" -ge 2 ] || ! status_is_paused_or_captain_held "$(last_status_line "$STATE/$(window_to_task "$w" "$STATE").status")"; }; then @@ -1073,6 +1117,7 @@ EOF if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then wedge_timer_check "$w" "$ssf" "busy (no completed turn)" "$ewf" else + wedge_shadow_resumed "$w" "$task" "$ssf" || true rm -f "$ssf" "$ewf" fi task=$(window_to_task "$w" "$STATE") diff --git a/bin/fm-wedge-score.py b/bin/fm-wedge-score.py new file mode 100755 index 00000000000..9d799c46be4 --- /dev/null +++ b/bin/fm-wedge-score.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# Usage: fm-wedge-score.py settle|score [options]. +# JSONL schema: {"ts":"","key":"","window":"","task":"","lane":"","idle_secs":,"outcome":"resumed|escalated"}. +# This script is the single owner of the wedge-settlements.jsonl schema. + +import argparse +import json +import math +import os +import sys +from datetime import datetime, timezone + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from argyle_gates import norm_cdf, norm_ppf + + +def _home(value): + return value or os.environ.get("FM_HOME", ".") + + +def _lane_for_task(home, task): + if not task: + return "unknown" + meta = os.path.join(home, "state", f"{task}.meta") + try: + with open(meta, encoding="utf-8") as handle: + for line in handle: + if line.startswith("project="): + return line.rstrip("\n").split("=", 1)[1] or "unknown" + except OSError: + pass + return "unknown" + + +def _settle(args): + try: + home = _home(args.home) + data_dir = os.path.join(home, "data") + os.makedirs(data_dir, exist_ok=True) + row = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "key": args.key, + "window": args.window, + "task": args.task, + "lane": _lane_for_task(home, args.task), + "idle_secs": args.idle_secs, + "outcome": args.outcome, + } + with open( + os.path.join(data_dir, "wedge-settlements.jsonl"), + "a", + encoding="utf-8", + ) as handle: + json.dump(row, handle, separators=(",", ":")) + handle.write("\n") + except Exception as exc: + print(f"fm-wedge-score settle: {exc}", file=sys.stderr) + return 0 + + +def _read_rows(home, lane): + path = os.path.join(home, "data", "wedge-settlements.jsonl") + rows = [] + try: + with open(path, encoding="utf-8") as handle: + for line in handle: + try: + row = json.loads(line) + except (TypeError, ValueError): + continue + if isinstance(row, dict) and row.get("lane") == lane: + rows.append(row) + except OSError: + pass + return rows + + +def _score(args): + rows = _read_rows(_home(args.home), args.lane) + healthy = [ + float(row["idle_secs"]) + for row in rows + if row.get("outcome") == "resumed" + ] + n = len(healthy) + n_total = len(rows) + n_esc = sum(row.get("outcome") == "escalated" for row in rows) + fixed_flag = args.idle_secs >= args.fixed_threshold + if n < args.min_obs: + result = { + "lane": args.lane, + "n": n, + "score": None, + "graded_flag": None, + "fixed_flag": fixed_flag, + "reason": f"insufficient data: {n}<{args.min_obs}", + } + else: + logs = [math.log(max(value, 1.0)) for value in healthy] + mu = sum(logs) / n + variance = sum((value - mu) ** 2 for value in logs) / (n - 1) + sigma = max(math.sqrt(variance), 0.25) + z = (math.log(max(args.idle_secs, 1)) - mu) / sigma + base_rate = (n_esc + 1) / (n_total + 2) + score = z - norm_ppf(1 - base_rate) + p_tail = 1 - norm_cdf(z) + result = { + "lane": args.lane, + "n": n, + "n_total": n_total, + "base_rate": round(base_rate, 4), + "z": round(z, 4), + "p_tail": round(p_tail, 4), + "score": round(score, 4), + "graded_flag": score > 0, + "fixed_flag": fixed_flag, + "reason": "ok", + } + print(json.dumps(result, separators=(",", ":"))) + return 0 + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--home", default=os.environ.get("FM_HOME", ".")) + subparsers = parser.add_subparsers(dest="command", required=True) + + settle = subparsers.add_parser("settle") + settle.add_argument("--home", default=None) + settle.add_argument("--key", required=True) + settle.add_argument("--window", required=True) + settle.add_argument("--task", default="") + settle.add_argument("--idle-secs", required=True, type=int) + settle.add_argument("--outcome", choices=("resumed", "escalated"), required=True) + settle.set_defaults(handler=_settle) + + score = subparsers.add_parser("score") + score.add_argument("--home", default=None) + score.add_argument("--lane", required=True) + score.add_argument("--idle-secs", required=True, type=int) + score.add_argument("--fixed-threshold", default=240, type=int) + score.add_argument("--min-obs", default=8, type=int) + score.set_defaults(handler=_score) + + args = parser.parse_args() + return args.handler(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fm-wedge-score.test.sh b/tests/fm-wedge-score.test.sh new file mode 100755 index 00000000000..c88a6a5c227 --- /dev/null +++ b/tests/fm-wedge-score.test.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash +# Regression tests for the fixed-timer wedge settlement shadow score. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found"; exit 0; } + +SCORE="$ROOT/bin/fm-wedge-score.py" +TMP_ROOT=$(fm_test_tmproot fm-wedge-score) + +test_settle_records_project_lane() { + local home out + home="$TMP_ROOT/project" + mkdir -p "$home/state" + printf 'project=demo\n' > "$home/state/task.meta" + "$SCORE" settle --home "$home" --key key --window sess:win --task task \ + --idle-secs 12 --outcome resumed + out=$(python3 - "$home/data/wedge-settlements.jsonl" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + row = json.loads(handle.readline()) +assert row["lane"] == "demo" +assert row["idle_secs"] == 12 +assert row["outcome"] == "resumed" +print("ok") +PY +) + [ "$out" = ok ] || fail "settle row validation failed" + pass "settle appends a row with the project lane" +} + +test_score_requires_minimum_observations() { + local home out + home="$TMP_ROOT/insufficient" + mkdir -p "$home/data" + python3 - "$home/data/wedge-settlements.jsonl" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for idle in (40, 50, 60): + handle.write(json.dumps({"lane": "demo", "idle_secs": idle, + "outcome": "resumed"}) + "\n") +PY + out=$("$SCORE" score --home "$home" --lane demo --idle-secs 100) + python3 - "$out" <<'PY' +import json +import sys +row = json.loads(sys.argv[1]) +assert row["score"] is None +assert row["reason"] == "insufficient data: 3<8" +PY + pass "score reports insufficient data below the minimum observations" +} + +test_score_flags_graded_and_fixed_thresholds() { + local home high low + home="$TMP_ROOT/scored" + mkdir -p "$home/data" + python3 - "$home/data/wedge-settlements.jsonl" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for idle in (40, 45, 50, 55, 60, 65, 70, 80, 90, 100): + handle.write(json.dumps({"lane": "demo", "idle_secs": idle, + "outcome": "resumed"}) + "\n") + handle.write(json.dumps({"lane": "demo", "idle_secs": 240, + "outcome": "escalated"}) + "\n") +PY + high=$("$SCORE" score --home "$home" --lane demo --idle-secs 600) + low=$("$SCORE" score --home "$home" --lane demo --idle-secs 50) + python3 - "$high" "$low" <<'PY' +import json +import sys +high, low = (json.loads(value) for value in sys.argv[1:]) +assert high["graded_flag"] is True +assert high["fixed_flag"] is True +assert high["score"] > 0 +assert low["graded_flag"] is False +PY + pass "score separates graded and fixed threshold flags" +} + +test_settle_missing_meta_uses_unknown_lane() { + local home out + home="$TMP_ROOT/missing-meta" + mkdir -p "$home" + "$SCORE" settle --home "$home" --key key --window sess:win --task missing \ + --idle-secs 4 --outcome escalated + out=$(python3 - "$home/data/wedge-settlements.jsonl" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + print(json.loads(handle.readline())["lane"]) +PY +) + [ "$out" = unknown ] || fail "missing meta lane was '$out'" + pass "settle uses unknown lane when task metadata is absent" +} + +test_settle_records_project_lane +test_score_requires_minimum_observations +test_score_flags_graded_and_fixed_thresholds +test_settle_missing_meta_uses_unknown_lane From 200241c959ae20a0b7d475aba810664cbf120899 Mon Sep 17 00:00:00 2001 From: Nathanael Roberts Date: Sat, 5 Sep 2026 08:38:32 -0700 Subject: [PATCH 2/4] no-mistakes(review): fix wedge shadow state resolution, thresholds, and tests --- bin/fm-watch.sh | 21 ++-- bin/fm-wedge-score.py | 96 +++++++++++------ docs/configuration.md | 1 + tests/fm-wedge-score.test.sh | 194 ++++++++++++++++++++++++++++------- 4 files changed, 232 insertions(+), 80 deletions(-) diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index e3abf037835..c2a431f0a5e 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -270,26 +270,21 @@ FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} # Shadow mode records wedge settlements and logs a graded score beside the fixed # timer, but never changes escalation. wedge_shadow_settle() { # - local win=$1 task=$2 idle=$3 outcome=$4 key + local win=$1 task=$2 idle=$3 outcome=$4 [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 command -v python3 >/dev/null 2>&1 || return 0 - key=$(printf '%s' "$win" | tr ':/.' '___') - "$SCRIPT_DIR/fm-wedge-score.py" settle --home "$FM_HOME" --key "$key" \ + "$SCRIPT_DIR/fm-wedge-score.py" settle --state "$STATE" \ --window "$win" --task "$task" --idle-secs "$idle" --outcome "$outcome" \ >/dev/null 2>&1 || true } -wedge_shadow_score() { # - local task=$2 idle=$3 lane=unknown meta score +wedge_shadow_score() { # + local task=$1 idle=$2 score [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 command -v python3 >/dev/null 2>&1 || return 0 - meta="$STATE/$task.meta" - if [ -n "$task" ] && [ -f "$meta" ]; then - lane=$(grep '^project=' "$meta" | cut -d= -f2- || true) - [ -n "$lane" ] || lane=unknown - fi - score=$("$SCRIPT_DIR/fm-wedge-score.py" score --home "$FM_HOME" \ - --lane "$lane" --idle-secs "$idle" 2>/dev/null) || true + score=$("$SCRIPT_DIR/fm-wedge-score.py" score --state "$STATE" \ + --task "$task" --idle-secs "$idle" \ + --fixed-threshold "$STALE_ESCALATE_SECS" 2>/dev/null) || true [ -n "$score" ] && triage_log "shadow wedge score: $score" || true } @@ -332,7 +327,7 @@ wedge_timer_check() { # ","key":"","window":"","task":"","lane":"","idle_secs":,"outcome":"resumed|escalated"}. -# This script is the single owner of the wedge-settlements.jsonl schema. +# Usage: fm-wedge-score.py settle|score --state [options]. +# JSONL schema: {"ts":"","window":"","task":"","lane":"","idle_secs":,"outcome":"resumed|escalated"}. +# This script is the single owner of the .wedge-settlements.jsonl schema, of the +# task-to-lane rule, and of the log's size bound. The log is watcher-owned +# bookkeeping and lives beside .watch-triage.log in the state directory the +# caller resolves. +# +# The graded score is a shadow comparator, not a calibrated one: its healthy +# sample is right-censored by the fixed timer it shadows (a pane idle past the +# threshold escalates instead of settling as resumed), so it never observes a +# legitimately long healthy tail. import argparse import json @@ -14,15 +22,14 @@ from argyle_gates import norm_cdf, norm_ppf +SETTLEMENTS = ".wedge-settlements.jsonl" +MAX_BYTES = 262144 -def _home(value): - return value or os.environ.get("FM_HOME", ".") - -def _lane_for_task(home, task): +def _lane_for_task(state, task): if not task: return "unknown" - meta = os.path.join(home, "state", f"{task}.meta") + meta = os.path.join(state, f"{task}.meta") try: with open(meta, encoding="utf-8") as handle: for line in handle: @@ -33,34 +40,46 @@ def _lane_for_task(home, task): return "unknown" +def _trim(path): + if os.path.getsize(path) < MAX_BYTES: + return + kept, size = [], 0 + with open(path, encoding="utf-8") as handle: + for line in reversed(handle.readlines()): + size += len(line.encode("utf-8")) + if size > MAX_BYTES // 2: + break + kept.append(line) + kept.reverse() + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as handle: + handle.writelines(kept) + os.replace(tmp, path) + + def _settle(args): try: - home = _home(args.home) - data_dir = os.path.join(home, "data") - os.makedirs(data_dir, exist_ok=True) + os.makedirs(args.state, exist_ok=True) row = { "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), - "key": args.key, "window": args.window, "task": args.task, - "lane": _lane_for_task(home, args.task), + "lane": _lane_for_task(args.state, args.task), "idle_secs": args.idle_secs, "outcome": args.outcome, } - with open( - os.path.join(data_dir, "wedge-settlements.jsonl"), - "a", - encoding="utf-8", - ) as handle: + path = os.path.join(args.state, SETTLEMENTS) + with open(path, "a", encoding="utf-8") as handle: json.dump(row, handle, separators=(",", ":")) handle.write("\n") + _trim(path) except Exception as exc: print(f"fm-wedge-score settle: {exc}", file=sys.stderr) return 0 -def _read_rows(home, lane): - path = os.path.join(home, "data", "wedge-settlements.jsonl") +def _read_rows(state, lane): + path = os.path.join(state, SETTLEMENTS) rows = [] try: with open(path, encoding="utf-8") as handle: @@ -76,20 +95,37 @@ def _read_rows(home, lane): return rows +def _escalation_incidents(rows): + # A pane that stays wedged re-escalates every threshold, so only the first + # escalation of a window since its last resume counts as an incident. + unresolved = set() + count = 0 + for row in rows: + window = row.get("window", "") + if row.get("outcome") == "escalated": + if window not in unresolved: + unresolved.add(window) + count += 1 + elif row.get("outcome") == "resumed": + unresolved.discard(window) + return count + + def _score(args): - rows = _read_rows(_home(args.home), args.lane) + lane = _lane_for_task(args.state, args.task) + rows = _read_rows(args.state, lane) healthy = [ float(row["idle_secs"]) for row in rows if row.get("outcome") == "resumed" ] n = len(healthy) - n_total = len(rows) - n_esc = sum(row.get("outcome") == "escalated" for row in rows) + n_esc = _escalation_incidents(rows) + n_total = n + n_esc fixed_flag = args.idle_secs >= args.fixed_threshold if n < args.min_obs: result = { - "lane": args.lane, + "lane": lane, "n": n, "score": None, "graded_flag": None, @@ -106,7 +142,7 @@ def _score(args): score = z - norm_ppf(1 - base_rate) p_tail = 1 - norm_cdf(z) result = { - "lane": args.lane, + "lane": lane, "n": n, "n_total": n_total, "base_rate": round(base_rate, 4), @@ -123,12 +159,10 @@ def _score(args): def main(): parser = argparse.ArgumentParser() - parser.add_argument("--home", default=os.environ.get("FM_HOME", ".")) subparsers = parser.add_subparsers(dest="command", required=True) settle = subparsers.add_parser("settle") - settle.add_argument("--home", default=None) - settle.add_argument("--key", required=True) + settle.add_argument("--state", required=True) settle.add_argument("--window", required=True) settle.add_argument("--task", default="") settle.add_argument("--idle-secs", required=True, type=int) @@ -136,10 +170,10 @@ def main(): settle.set_defaults(handler=_settle) score = subparsers.add_parser("score") - score.add_argument("--home", default=None) - score.add_argument("--lane", required=True) + score.add_argument("--state", required=True) + score.add_argument("--task", default="") score.add_argument("--idle-secs", required=True, type=int) - score.add_argument("--fixed-threshold", default=240, type=int) + score.add_argument("--fixed-threshold", required=True, type=int) score.add_argument("--min-obs", default=8, type=int) score.set_defaults(handler=_score) diff --git a/docs/configuration.md b/docs/configuration.md index 1d1c121ed7a..b42f0c14269 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -573,6 +573,7 @@ FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stal FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added +FM_WEDGE_SHADOW=1 # set to 0 to stop recording wedge settlements in state/.wedge-settlements.jsonl and logging the shadow graded score beside the fixed timer; the score is observational only (its healthy sample is right-censored by FM_STALE_ESCALATE_SECS) and never changes escalation FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's absorbed-wake debug log FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT= # optional seconds allowed for bootstrap's best-effort clone refresh; unset/blank defaults to max(20, 5 + 3 * origin-backed-project-count) FM_FLEET_PRUNE=1 # set to 0 to skip pruning local branches whose upstream is gone diff --git a/tests/fm-wedge-score.test.sh b/tests/fm-wedge-score.test.sh index c88a6a5c227..541c606876d 100755 --- a/tests/fm-wedge-score.test.sh +++ b/tests/fm-wedge-score.test.sh @@ -9,22 +9,24 @@ command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found"; exit 0; SCORE="$ROOT/bin/fm-wedge-score.py" TMP_ROOT=$(fm_test_tmproot fm-wedge-score) +LOG=.wedge-settlements.jsonl test_settle_records_project_lane() { - local home out - home="$TMP_ROOT/project" - mkdir -p "$home/state" - printf 'project=demo\n' > "$home/state/task.meta" - "$SCORE" settle --home "$home" --key key --window sess:win --task task \ + local state out + state="$TMP_ROOT/project/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + "$SCORE" settle --state "$state" --window sess:win --task task \ --idle-secs 12 --outcome resumed - out=$(python3 - "$home/data/wedge-settlements.jsonl" <<'PY' + out=$(python3 - "$state/$LOG" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as handle: row = json.loads(handle.readline()) -assert row["lane"] == "demo" -assert row["idle_secs"] == 12 -assert row["outcome"] == "resumed" +assert row["lane"] == "demo", row +assert row["idle_secs"] == 12, row +assert row["outcome"] == "resumed", row +assert row["window"] == "sess:win", row print("ok") PY ) @@ -33,63 +35,146 @@ PY } test_score_requires_minimum_observations() { - local home out - home="$TMP_ROOT/insufficient" - mkdir -p "$home/data" - python3 - "$home/data/wedge-settlements.jsonl" <<'PY' + local state out check + state="$TMP_ROOT/insufficient/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + python3 - "$state/$LOG" <<'PY' import json import sys with open(sys.argv[1], "w", encoding="utf-8") as handle: for idle in (40, 50, 60): - handle.write(json.dumps({"lane": "demo", "idle_secs": idle, + handle.write(json.dumps({"lane": "demo", "window": "s:w", + "idle_secs": idle, "outcome": "resumed"}) + "\n") PY - out=$("$SCORE" score --home "$home" --lane demo --idle-secs 100) - python3 - "$out" <<'PY' + out=$("$SCORE" score --state "$state" --task task --idle-secs 100 \ + --fixed-threshold 240) + check=$(python3 - "$out" <<'PY' import json import sys row = json.loads(sys.argv[1]) -assert row["score"] is None -assert row["reason"] == "insufficient data: 3<8" +assert row["score"] is None, row +assert row["graded_flag"] is None, row +assert row["reason"] == "insufficient data: 3<8", row +print("ok") PY +) || fail "insufficient-data assertions failed for $out" + [ "$check" = ok ] || fail "insufficient-data validation failed for $out" pass "score reports insufficient data below the minimum observations" } test_score_flags_graded_and_fixed_thresholds() { - local home high low - home="$TMP_ROOT/scored" - mkdir -p "$home/data" - python3 - "$home/data/wedge-settlements.jsonl" <<'PY' + local state high low check + state="$TMP_ROOT/scored/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + python3 - "$state/$LOG" <<'PY' import json import sys with open(sys.argv[1], "w", encoding="utf-8") as handle: for idle in (40, 45, 50, 55, 60, 65, 70, 80, 90, 100): - handle.write(json.dumps({"lane": "demo", "idle_secs": idle, + handle.write(json.dumps({"lane": "demo", "window": "s:w", + "idle_secs": idle, "outcome": "resumed"}) + "\n") - handle.write(json.dumps({"lane": "demo", "idle_secs": 240, + handle.write(json.dumps({"lane": "demo", "window": "s:w", + "idle_secs": 240, "outcome": "escalated"}) + "\n") PY - high=$("$SCORE" score --home "$home" --lane demo --idle-secs 600) - low=$("$SCORE" score --home "$home" --lane demo --idle-secs 50) - python3 - "$high" "$low" <<'PY' + high=$("$SCORE" score --state "$state" --task task --idle-secs 600 \ + --fixed-threshold 240) + low=$("$SCORE" score --state "$state" --task task --idle-secs 50 \ + --fixed-threshold 240) + check=$(python3 - "$high" "$low" <<'PY' import json import sys high, low = (json.loads(value) for value in sys.argv[1:]) -assert high["graded_flag"] is True -assert high["fixed_flag"] is True -assert high["score"] > 0 -assert low["graded_flag"] is False +assert high["graded_flag"] is True, high +assert high["fixed_flag"] is True, high +assert high["score"] > 0, high +assert low["graded_flag"] is False, low +assert low["fixed_flag"] is False, low +print("ok") PY +) || fail "graded/fixed assertions failed for high=$high low=$low" + [ "$check" = ok ] || fail "graded/fixed validation failed for high=$high low=$low" pass "score separates graded and fixed threshold flags" } +# The escalation bound is FM_STALE_ESCALATE_SECS, not a constant, so the caller +# supplies it and fixed_flag reports what the fixed timer actually did. +test_fixed_flag_follows_the_supplied_threshold() { + local state out check + state="$TMP_ROOT/threshold/state" + mkdir -p "$state" + out=$("$SCORE" score --state "$state" --task missing --idle-secs 120 \ + --fixed-threshold 120) + check=$(python3 - "$out" <<'PY' +import json +import sys +row = json.loads(sys.argv[1]) +assert row["fixed_flag"] is True, row +print("ok") +PY +) || fail "fixed_flag did not follow the supplied threshold: $out" + [ "$check" = ok ] || fail "fixed_flag validation failed for $out" + pass "fixed_flag follows the caller's escalation threshold" +} + +# A pane that stays wedged re-escalates every threshold; counting those repeats +# as separate incidents would inflate the base rate and lower the graded bar. +test_repeated_escalations_count_as_one_incident() { + local state repeated distinct check + state="$TMP_ROOT/incidents/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for idle in (40, 45, 50, 55, 60, 65, 70, 80): + handle.write(json.dumps({"lane": "demo", "window": "s:healthy", + "idle_secs": idle, + "outcome": "resumed"}) + "\n") + for _ in range(6): + handle.write(json.dumps({"lane": "demo", "window": "s:wedged", + "idle_secs": 240, + "outcome": "escalated"}) + "\n") +PY + repeated=$("$SCORE" score --state "$state" --task task --idle-secs 300 \ + --fixed-threshold 240) + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "a", encoding="utf-8") as handle: + for window in ("s:a", "s:b", "s:c", "s:d", "s:e"): + handle.write(json.dumps({"lane": "demo", "window": window, + "idle_secs": 240, + "outcome": "escalated"}) + "\n") +PY + distinct=$("$SCORE" score --state "$state" --task task --idle-secs 300 \ + --fixed-threshold 240) + check=$(python3 - "$repeated" "$distinct" <<'PY' +import json +import sys +repeated, distinct = (json.loads(value) for value in sys.argv[1:]) +assert repeated["n_total"] == 9, repeated +assert distinct["n_total"] == 14, distinct +assert distinct["base_rate"] > repeated["base_rate"], (repeated, distinct) +print("ok") +PY +) || fail "incident counting failed for repeated=$repeated distinct=$distinct" + [ "$check" = ok ] || fail "incident validation failed for repeated=$repeated distinct=$distinct" + pass "repeated escalations of one window count as a single incident" +} + test_settle_missing_meta_uses_unknown_lane() { - local home out - home="$TMP_ROOT/missing-meta" - mkdir -p "$home" - "$SCORE" settle --home "$home" --key key --window sess:win --task missing \ + local state out + state="$TMP_ROOT/missing-meta/state" + mkdir -p "$(dirname "$state")" + "$SCORE" settle --state "$state" --window sess:win --task missing \ --idle-secs 4 --outcome escalated - out=$(python3 - "$home/data/wedge-settlements.jsonl" <<'PY' + out=$(python3 - "$state/$LOG" <<'PY' import json import sys with open(sys.argv[1], encoding="utf-8") as handle: @@ -100,7 +185,44 @@ PY pass "settle uses unknown lane when task metadata is absent" } +test_settlement_log_stays_bounded() { + local state before after check + state="$TMP_ROOT/bounded/state" + mkdir -p "$state" + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for i in range(4000): + handle.write(json.dumps({"ts": "2026-01-01T00:00:00Z", "window": "s:w", + "task": "old", "lane": "demo", + "idle_secs": i, "outcome": "resumed"}) + "\n") +PY + before=$(wc -c < "$state/$LOG" | tr -d '[:space:]') + [ "$before" -gt 262144 ] || fail "fixture log was only $before bytes" + "$SCORE" settle --state "$state" --window sess:win --task newest \ + --idle-secs 7 --outcome escalated + after=$(wc -c < "$state/$LOG" | tr -d '[:space:]') + [ "$after" -lt "$before" ] || fail "settlement log was not trimmed: $after bytes" + check=$(python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + rows = [json.loads(line) for line in handle if line.strip()] +assert rows, "trimming emptied the log" +assert rows[-1]["task"] == "newest", rows[-1] +assert rows[-1]["idle_secs"] == 7, rows[-1] +print("ok") +PY +) || fail "the newest settlement did not survive trimming" + [ "$check" = ok ] || fail "trimmed log validation failed" + pass "the settlement log is trimmed once it passes its size cap" +} + test_settle_records_project_lane test_score_requires_minimum_observations test_score_flags_graded_and_fixed_thresholds +test_fixed_flag_follows_the_supplied_threshold +test_repeated_escalations_count_as_one_incident test_settle_missing_meta_uses_unknown_lane +test_settlement_log_stays_bounded From 80f493d2f891042cfb61eabf743c897dca2cf610 Mon Sep 17 00:00:00 2001 From: Nathanael Roberts Date: Sat, 5 Sep 2026 09:03:07 -0700 Subject: [PATCH 3/4] no-mistakes(document): clarify triage log role for non-absorb watcher lines --- AGENTS.md | 2 +- bin/fm-watch-arm.sh | 4 ++-- docs/architecture.md | 2 +- docs/configuration.md | 2 +- docs/watcher-continuity.md | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 896847a3ea8..8276ad0bc88 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,7 +118,7 @@ state/ volatile runtime signals; gitignored .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch - .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete + .watch-triage.log watcher's observational debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch .no-mistakes/ local validation state and evidence; gitignored diff --git a/bin/fm-watch-arm.sh b/bin/fm-watch-arm.sh index 81c09098d84..9b562c00aaf 100755 --- a/bin/fm-watch-arm.sh +++ b/bin/fm-watch-arm.sh @@ -48,8 +48,8 @@ # state/.watch-cycle-exits.log. The arm layer owns that bounded ledger; it records # arm/watcher identities, timestamps, exit/signal classification, beacon age, # lock identity before and after close, and successor disposition. The separate -# state/.watch-triage.log remains exclusively the watcher's absorbed-wake debug -# log and is never written here. +# state/.watch-triage.log is the watcher's own bounded observational debug log, +# carries no lifecycle semantics, and is never written here. # # --restart: stop ONLY this FM_HOME's watcher (the pid recorded in THIS home's # state/.watch.lock) and own a fresh cycle, or attach if a verified live peer diff --git a/docs/architecture.md b/docs/architecture.md index e56bacdc2ba..d5fc00b7434 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,7 +60,7 @@ At session start, `bin/fm-session-start.sh` emits exactly one primary-harness su That block owns the live wait shape for the running primary harness: Claude's Stop `asyncRewake` hook owns tokenless re-arm cycles, Grok uses background-notify cycles, Codex uses bounded foreground checkpoints, Pi and pi-signed use the same two tracked primary extensions, and OpenCode uses its TUI plugin. `bin/fm-watch-arm.sh` remains the verified arm wrapper for protocols that call it; it forks the watcher as a tracked child, verifies it is genuinely alive with a fresh liveness beacon, and prints an honest `started`, `attached`, or nonzero `FAILED` status. [`watcher-continuity.md`](watcher-continuity.md#arm-layer-cycle-contract) owns the arm layer's successor, terminal-delivery, and typed clean-close failure contract. -The arm layer records one bounded lifecycle row per observed cycle in `state/.watch-cycle-exits.log`; `state/.watch-triage.log` remains exclusively the absorbed-wake debug log. +The arm layer records one bounded lifecycle row per observed cycle in `state/.watch-cycle-exits.log`, which is the only watcher log with lifecycle semantics. Pi and OpenCode verify session-lock ownership and launch one singleton successor from their child-close handlers before delivering an actionable wake prompt, with bounded exponential retry for failed restoration. Claude's `bin/fm-claude-stop-autoarm.sh` hook fires on every Stop and, when the home is eligible and still needs supervision, claims one home-scoped cycle, foregrounds the arm wrapper, and translates actionable closes into exit-2 rewakes. It suppresses failed-looking closes when the same identity-matched watcher is healthy, retries genuine failures within a bound, and coordinates exhausted failure episodes with the Claude turn-end guard as documented in [`turnend-guard.md`](turnend-guard.md). diff --git a/docs/configuration.md b/docs/configuration.md index b42f0c14269..f920d471d5a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -574,7 +574,7 @@ FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/< FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added FM_WEDGE_SHADOW=1 # set to 0 to stop recording wedge settlements in state/.wedge-settlements.jsonl and logging the shadow graded score beside the fixed timer; the score is observational only (its healthy sample is right-censored by FM_STALE_ESCALATE_SECS) and never changes escalation -FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's absorbed-wake debug log +FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's bounded observational debug log (absorbed wakes and other non-actionable notes) FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT= # optional seconds allowed for bootstrap's best-effort clone refresh; unset/blank defaults to max(20, 5 + 3 * origin-backed-project-count) FM_FLEET_PRUNE=1 # set to 0 to skip pruning local branches whose upstream is gone FM_STALE_WORKTREE_LOCK_AGE_SECS=30 # min mtime age before fm-teardown.sh treats a leftover worktree git index.lock as provably stale diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 2ae9a6b17bb..b73b8483b8a 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -53,7 +53,7 @@ Only a cycle with no matching delivery record emits `watcher: FAILED - cycle end The arm layer appends one tab-separated record per observed cycle to `state/.watch-cycle-exits.log`. Each record includes arm and watcher PIDs, start and end timestamps, exit code and signal, classified reason, beacon age, lock identity before and after close, and successor disposition. The file is size-capped through `FM_WATCH_CYCLE_LOG_MAX_BYTES` and `FM_WATCH_CYCLE_LOG_KEEP_LINES`. -`state/.watch-triage.log` remains only the watcher's bounded absorbed-wake debug log and carries no lifecycle semantics. +`state/.watch-triage.log` remains only the watcher's own bounded observational debug log, written for absorbed wakes and other non-actionable watcher notes, and carries no lifecycle semantics. The default 300-second grace is unchanged. Only the watcher process touches `state/.last-watcher-beat`; no helper process can make a wedged watcher appear healthy. From 9f7a30c17e4623e56e9b9615079d2da9eae15a0f Mon Sep 17 00:00:00 2001 From: Nathanael Roberts Date: Sat, 5 Sep 2026 09:56:38 -0700 Subject: [PATCH 4/4] =?UTF-8?q?no-mistakes(ci):=20Failing=20check=20"Behav?= =?UTF-8?q?ior=20portable=20serial=204"=20was=20tests/fm-pi-watch-extensio?= =?UTF-8?q?n.test.sh,=20assertion=20"Pi=20must=20deliver=20the=20actionabl?= =?UTF-8?q?e=20wake=20after=20bounded=20hung-successor=20recovery"=20(conf?= =?UTF-8?q?irmed=20by=20pulling=20the=20full=20job=20log=20for=20check=20r?= =?UTF-8?q?un=20101334942108;=20the=20provided=20excerpt=20started=20after?= =?UTF-8?q?=20that=20test).=20It=20is=20a=20timing=20flake=20in=20the=20te?= =?UTF-8?q?st,=20not=20a=20defect=20in=20this=20PR's=20code:=20the=20PR's?= =?UTF-8?q?=20only=20watcher-family=20change=20to=20bin/fm-watch-arm.sh=20?= =?UTF-8?q?is=20a=20comment,=20and=20the=20test=20substitutes=20its=20own?= =?UTF-8?q?=20fm-watch-arm.sh=20fixture.=20Root=20cause:=20the=20test=20se?= =?UTF-8?q?t=20FM=5FPI=5FARM=5FREADY=5FTIMEOUT=5FMS=3D250.=20That=20budget?= =?UTF-8?q?=20bounds=20a=20successor=20that=20never=20reports=20ready,=20s?= =?UTF-8?q?o=20it=20also=20bounds=20how=20long=20the=20spawned=20`bash=20-?= =?UTF-8?q?lc=20...=20exec=20fm-watch-arm.sh`=20child=20has=20to=20append?= =?UTF-8?q?=20its=20`arm=3D`=20row.=20Under=20CI=20load=20a=20cold=20?= =?UTF-8?q?login=20shell=20exceeds=20it,=20the=20extension=20SIGTERMs=20th?= =?UTF-8?q?e=20attempt=20before=20the=20script=20runs,=20the=20row=20is=20?= =?UTF-8?q?lost,=20and=20the=20exact-count=20assertion=20("expected=20one?= =?UTF-8?q?=20successor=20plus=20two=20retries,=20got=20N")=20fails.=20Rep?= =?UTF-8?q?roduced=20locally=20on=20the=20sibling=20OpenCode=20case=20by?= =?UTF-8?q?=20lowering=20the=20budget=20to=2030ms,=20yielding=20exactly=20?= =?UTF-8?q?the=20CI-shaped=20failure.=20Fix=20(tests/fm-pi-watch-extension?= =?UTF-8?q?.test.sh=20only):=20raised=20the=20readiness=20budget=20250ms?= =?UTF-8?q?=20->=202000ms=20in=20both=20hung-successor=20cases=20(Pi=20and?= =?UTF-8?q?=20OpenCode=20=E2=80=94=20identical=20fixture=20and=20defect),?= =?UTF-8?q?=20raised=20the=20prompt-wait=20poll=20bound=20500=20->=202000?= =?UTF-8?q?=20iterations=20(5s=20->=2020s)=20so=20it=20still=20covers=20th?= =?UTF-8?q?e=20longer=203=20x=202s=20recovery,=20and=20added=20a=20comment?= =?UTF-8?q?=20at=20both=20sites=20explaining=20the=20budget=20must=20stay?= =?UTF-8?q?=20far=20above=20child=20start-up.=20No=20production=20code=20c?= =?UTF-8?q?hanged,=20so=20the=20fixed=20240s=20escalation=20and=20shadow?= =?UTF-8?q?=20wedge=20plumbing=20are=20untouched.=20Verification:=20Pi=20h?= =?UTF-8?q?ung-successor=203/3=20green,=20OpenCode=20hung-successor=203/3?= =?UTF-8?q?=20green,=20all=20Pi=20tests=20and=20all=20remaining=20OpenCode?= =?UTF-8?q?=20tests=20in=20the=20file=20green=20except=20two=20that=20fail?= =?UTF-8?q?=20in=20this=20sandbox=20for=20environmental=20reasons=20unrela?= =?UTF-8?q?ted=20to=20the=20change=20(OpenCode=20session-lock=20and=20turn?= =?UTF-8?q?-end-guard=20cases;=20both=20pass=20in=20CI=20on=20this=20branc?= =?UTF-8?q?h's=20earlier=20green=20run).=20`bash=20-n`=20clean;=20ShellChe?= =?UTF-8?q?ck=20is=20not=20installed=20here=20so=20bin/fm-lint.sh=20was=20?= =?UTF-8?q?skipped.=20Also=20deleted=20a=20stray=20untracked=20scratch=20f?= =?UTF-8?q?ile=20tests/ztmp-hung.sh=20left=20in=20the=20worktree?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/fm-pi-watch-extension.test.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/tests/fm-pi-watch-extension.test.sh b/tests/fm-pi-watch-extension.test.sh index f8883194898..cee8f8dee68 100755 --- a/tests/fm-pi-watch-extension.test.sh +++ b/tests/fm-pi-watch-extension.test.sh @@ -417,7 +417,11 @@ trap 'exit 0' TERM INT while :; do sleep 0.02; done SH chmod +x "$repo/bin/fm-watch-arm.sh" - out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_PI_ARM_READY_TIMEOUT_MS=250 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node --input-type=module 2>&1 <<'EOF' + # The readiness budget bounds a successor that never reports ready, so it + # also bounds how long the spawned login-shell arm child has to record its + # row. Sized near cold `bash -l` start-up cost it retires an attempt before + # the script runs and silently loses an arm row, so keep it far above that. + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_PI_ARM_READY_TIMEOUT_MS=2000 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node --input-type=module 2>&1 <<'EOF' import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; @@ -441,7 +445,7 @@ writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); const mod = await import(pathToFileURL(process.env.PLUGIN).href); mod.default(pi); await tool.execute("tool-call-hung-successor", {}, undefined, undefined, {}); -for (let i = 0; i < 500 && !prompt; i += 1) { +for (let i = 0; i < 2000 && !prompt; i += 1) { await new Promise((resolve) => setTimeout(resolve, 10)); } const rows = existsSync(process.env.FM_ARM_LOG) @@ -1579,7 +1583,11 @@ trap 'exit 0' TERM INT while :; do sleep 0.02; done SH chmod +x "$repo/bin/fm-watch-arm.sh" - out=$(PLUGIN="$plugin" WORKTREE="$repo" FM_HOME="$home" FM_ARM_LOG="$log" FM_OPENCODE_ARM_READY_TIMEOUT_MS=250 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node 2>&1 <<'EOF' + # The readiness budget bounds a successor that never reports ready, so it + # also bounds how long the spawned login-shell arm child has to record its + # row. Sized near cold `bash -l` start-up cost it retires an attempt before + # the script runs and silently loses an arm row, so keep it far above that. + out=$(PLUGIN="$plugin" WORKTREE="$repo" FM_HOME="$home" FM_ARM_LOG="$log" FM_OPENCODE_ARM_READY_TIMEOUT_MS=2000 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node 2>&1 <<'EOF' import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; @@ -1603,7 +1611,7 @@ const hooks = await mod.FmPrimaryWatchArm({ }); writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); await hooks.event({ event: { type: "session.idle", properties: { sessionID: "session-test" } } }); -for (let i = 0; i < 500 && !prompt; i += 1) { +for (let i = 0; i < 2000 && !prompt; i += 1) { await new Promise((resolve) => setTimeout(resolve, 10)); } const rows = existsSync(process.env.FM_ARM_LOG)