diff --git a/AGENTS.md b/AGENTS.md index 896847a3ea8..8276ad0bc88 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -118,7 +118,7 @@ state/ volatile runtime signals; gitignored .watch.lock .wake-queue.lock watcher singleton and queue serialization locks .claude-autoarm.lock .claude-autoarm-epoch .claude-autoarm-failure-notified .claude-autoarm-failure-alarmed .turnend-claude-blocks .turnend-claude-blocks.lock Claude Stop auto-arm single-flight, epoch, failure-episode, attended-alarm, guard-budget, and budget-lock records; never touch .hash-* .count-* .stale-* .stale-since-* .paused-* .wedge-escalations-* .seen-* .hb-surfaced-* .last-* .heartbeat-streak watcher internals; never touch - .watch-triage.log watcher's absorbed-wake debug log (size-capped); never relied on, safe to delete + .watch-triage.log watcher's observational debug log (size-capped); never relied on, safe to delete .last-watcher-beat watcher liveness beacon, touched every poll (including while absorbing benign wakes); guard scripts read it .subsuper-* .supervise-daemon.* sub-supervisor internals; never touch .no-mistakes/ local validation state and evidence; gitignored diff --git a/bin/argyle_gates.py b/bin/argyle_gates.py new file mode 100644 index 00000000000..711308bf57d --- /dev/null +++ b/bin/argyle_gates.py @@ -0,0 +1,241 @@ +# Provenance: lifted verbatim from cipherholdingsllc/nakatomi +# branch t2-argyle-frontier, path crypto-crawler/crypto_crawler/argyle/gates.py +# (commit 02be13ca139e847089d3ce7c6503d9f5dae914ba). +# Pure stdlib. Do not edit here; upstream fixes land in nakatomi first. +""" +Promotion gates — the bar every learned knob must clear before it may touch +the mock book. Three tests, ALL required (logical AND): + + 1. Deflated Sharpe (Bailey & López de Prado 2014): the observed Sharpe must + be significant AFTER deflating for the number of configurations actually + tried. Sweeping N configs and reporting the best is the quant original + sin; the deflation term grows with N, so a small sample cannot launder a + lucky config into a promotion. + 2. Block-permutation test: the strategy's edge statistic must beat p < 0.05 + against block-shuffled nulls (blocks preserve short-range autocorrelation + that a naive i.i.d. shuffle would destroy — shuffling away autocorrelation + makes nulls too easy to beat). + 3. Out-of-sample calibration: mean Brier of the candidate's p_up on held-out + outcomes must beat CLIMATOLOGY — the constant base-rate predictor, whose + Brier is p̄(1−p̄) ≤ 0.25. A fixed 0.25 bar is launderable at skewed base + rates (constant p=0.9 on 90%-up outcomes scores 0.09 with zero skill — + found by adversarial audit 2026-07-04); climatology is not. + +Everything is pure Python + math (no scipy — this box is minimal, and a gate +must not fail-open because a dependency is missing). Normal CDF via math.erf; +inverse normal via Acklam's rational approximation (|ε| < 1.15e-9, far below +any decision boundary here). + +Honest-refusal posture: with too little data the gate returns passed=False +with reason "insufficient data" — it never extrapolates a pass. On ~teens of +settled findings the EXPECTED outcome is failure; that is the gate working. +""" + +from __future__ import annotations + +import math +import random +from dataclasses import dataclass, field + +EULER_GAMMA = 0.5772156649015329 + + +def norm_cdf(x: float) -> float: + return 0.5 * (1.0 + math.erf(x / math.sqrt(2.0))) + + +def norm_ppf(p: float) -> float: + """Acklam's inverse-normal approximation. Pure, dependency-free.""" + if not 0.0 < p < 1.0: + raise ValueError(f"p must be in (0,1), got {p}") + a = (-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, + 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00) + b = (-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, + 6.680131188771972e+01, -1.328068155288572e+01) + c = (-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, + -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00) + d = (7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, + 3.754408661907416e+00) + p_low, p_high = 0.02425, 1 - 0.02425 + if p < p_low: + q = math.sqrt(-2 * math.log(p)) + return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / \ + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + if p > p_high: + q = math.sqrt(-2 * math.log(1 - p)) + return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / \ + ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1) + q = p - 0.5 + r = q * q + return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / \ + (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1) + + +def _moments(xs: list[float]) -> tuple[float, float, float, float]: + """(mean, std, skew, kurtosis) — kurtosis is NON-excess (normal = 3.0), + matching the PSR formula's convention. Sample std (ddof=1).""" + n = len(xs) + mean = sum(xs) / n + var = sum((x - mean) ** 2 for x in xs) / (n - 1) + std = math.sqrt(var) + if std == 0.0: + return mean, 0.0, 0.0, 3.0 + skew = sum(((x - mean) / std) ** 3 for x in xs) / n + kurt = sum(((x - mean) / std) ** 4 for x in xs) / n + return mean, std, skew, kurt + + +def sharpe(returns: list[float]) -> float: + """Per-period, non-annualized. Sample std.""" + mean, std, _, _ = _moments(returns) + if std == 0.0: + return 0.0 + return mean / std + + +@dataclass +class GateResult: + passed: bool + reason: str + detail: dict = field(default_factory=dict) + + +def deflated_sharpe( + returns: list[float], + n_trials: int, + var_sr_trials: float | None = None, + confidence: float = 0.95, +) -> GateResult: + """DSR = PSR evaluated at the expected-max-Sharpe benchmark under n_trials. + + SR0 = sqrt(V[SR_n]) * ((1-γ)·Φ⁻¹(1-1/N) + γ·Φ⁻¹(1-1/(N·e))) + DSR = Φ( (SR - SR0)·sqrt(T-1) / sqrt(1 - γ₃·SR + (γ₄-1)/4·SR²) ) + + `var_sr_trials` is the cross-trial variance of the Sharpe estimates from + the actual config sweep — pass it when the sweep recorded per-config SRs. + When unavailable, the estimator's own asymptotic variance is used as a + stand-in (documented compromise: it under-deflates a heterogeneous sweep, + so prefer real trial variance for any promotion decision).""" + t = len(returns) + if t < 10: + return GateResult(False, f"insufficient data: {t} returns < 10") + if n_trials < 1: + return GateResult(False, f"n_trials must be >= 1, got {n_trials}") + sr = sharpe(returns) + _, _, skew, kurt = _moments(returns) + denom = 1.0 - skew * sr + (kurt - 1.0) / 4.0 * sr * sr + if denom <= 0.0: + return GateResult(False, "PSR variance term non-positive (pathological moments)", + {"sr": sr, "skew": skew, "kurt": kurt}) + if var_sr_trials is None: + var_sr_trials = denom / (t - 1) + if n_trials == 1: + sr0 = 0.0 + else: + e = math.e + sr0 = math.sqrt(max(var_sr_trials, 0.0)) * ( + (1 - EULER_GAMMA) * norm_ppf(1 - 1.0 / n_trials) + + EULER_GAMMA * norm_ppf(1 - 1.0 / (n_trials * e)) + ) + dsr = norm_cdf((sr - sr0) * math.sqrt(t - 1) / math.sqrt(denom)) + passed = dsr >= confidence + return GateResult( + passed, + f"DSR={dsr:.4f} {'≥' if passed else '<'} {confidence} (SR={sr:.4f}, SR0={sr0:.4f}, " + f"N={n_trials}, T={t})", + {"dsr": dsr, "sr": sr, "sr0": sr0, "n_trials": n_trials, "t": t}, + ) + + +def block_permutation( + directions: list[float], + returns: list[float], + n_perm: int = 2000, + block: int | None = None, + seed: int = 0, + alpha: float = 0.05, +) -> GateResult: + """H0: directions carry no information about returns. Statistic is the + mean directional capture mean(d_i·r_i). Null draws are a CIRCULAR BLOCK + BOOTSTRAP of the return series (blocks drawn with replacement, order + within blocks preserved) — short-range autocorrelation survives, and the + p-value is approximate rather than an exact permutation p. Seeded — same + inputs, same p. Add-one estimator (Phipson & Smyth) so p is never 0.""" + n = len(returns) + if n != len(directions): + return GateResult(False, "directions/returns length mismatch") + if n < 8: + return GateResult(False, f"insufficient data: {n} pairs < 8") + if block is None: + block = max(1, round(n ** (1.0 / 3.0))) # standard b ~ n^(1/3) rule + obs = sum(d * r for d, r in zip(directions, returns)) / n + rng = random.Random(seed) + n_blocks = math.ceil(n / block) + hits = 0 + for _ in range(n_perm): + starts = [rng.randrange(n) for _ in range(n_blocks)] + permuted: list[float] = [] + for s in starts: + for j in range(block): + if len(permuted) == n: + break + permuted.append(returns[(s + j) % n]) + stat = sum(d * r for d, r in zip(directions, permuted)) / n + if stat >= obs: + hits += 1 + p = (hits + 1) / (n_perm + 1) + passed = p < alpha + return GateResult( + passed, + f"block-permutation p={p:.4f} {'<' if passed else '≥'} {alpha} " + f"(obs={obs:.6f}, block={block}, n_perm={n_perm})", + {"p": p, "obs": obs, "block": block, "n_perm": n_perm}, + ) + + +def oos_calibration(p_up: list[float], outcomes: list[int]) -> GateResult: + """Mean Brier of held-out p_up must beat the climatology Brier p̄(1−p̄) — + the score of constantly predicting the realized base rate. This is the + skill bar a constant predictor cannot pass by construction.""" + if len(p_up) != len(outcomes): + return GateResult(False, "p_up/outcomes length mismatch") + if len(p_up) < 10: + return GateResult(False, f"insufficient data: {len(p_up)} pairs < 10") + brier = sum((p - o) ** 2 for p, o in zip(p_up, outcomes)) / len(p_up) + base_rate = sum(outcomes) / len(outcomes) + climatology = base_rate * (1.0 - base_rate) + passed = brier < climatology + return GateResult( + passed, + f"OOS Brier={brier:.4f} {'<' if passed else '≥'} climatology " + f"{climatology:.4f} (p̄={base_rate:.3f}, n={len(p_up)}, coin-flip bound 0.25)", + {"brier": brier, "n": len(p_up), "climatology": climatology, + "base_rate": base_rate, "coin_flip": 0.25}, + ) + + +def promotion_gate( + returns: list[float], + n_trials: int, + directions: list[float], + p_up: list[float], + outcomes: list[int], + var_sr_trials: float | None = None, + seed: int = 0, +) -> GateResult: + """The full gate: deflated Sharpe AND block permutation AND OOS calibration. + Returns one GateResult whose detail carries all three components — this + dict IS the promotion receipt body.""" + dsr = deflated_sharpe(returns, n_trials, var_sr_trials) + perm = block_permutation(directions, returns, seed=seed) + cal = oos_calibration(p_up, outcomes) + passed = dsr.passed and perm.passed and cal.passed + reason = " | ".join( + f"{name}: {'PASS' if g.passed else 'FAIL'} ({g.reason})" + for name, g in (("deflated_sharpe", dsr), ("block_permutation", perm), ("oos_calibration", cal)) + ) + return GateResult(passed, reason, { + "deflated_sharpe": {"passed": dsr.passed, **dsr.detail}, + "block_permutation": {"passed": perm.passed, **perm.detail}, + "oos_calibration": {"passed": cal.passed, **cal.detail}, + }) diff --git a/bin/fm-test-run.sh b/bin/fm-test-run.sh index 9320178e41e..7a15f463bca 100755 --- a/bin/fm-test-run.sh +++ b/bin/fm-test-run.sh @@ -141,6 +141,7 @@ family_for_basename() { fm-kimi-harness.test.sh|fm-muse-harness.test.sh|fm-herdr-lab.test.sh|fm-lint.test.sh|\ fm-operational-input.test.sh|fm-pi-primary-types.test.sh|\ fm-send-popup-settle.test.sh|fm-send-settle.test.sh|\ + fm-wedge-score.test.sh|\ fm-subagent-pretool-check.test.sh|\ fm-supervision-instructions.test.sh|fm-task-delivery.test.sh|\ fm-tmux-submit-busy.test.sh|fm-trace-context-lib.test.sh|\ diff --git a/bin/fm-watch-arm.sh b/bin/fm-watch-arm.sh index 81c09098d84..9b562c00aaf 100755 --- a/bin/fm-watch-arm.sh +++ b/bin/fm-watch-arm.sh @@ -48,8 +48,8 @@ # state/.watch-cycle-exits.log. The arm layer owns that bounded ledger; it records # arm/watcher identities, timestamps, exit/signal classification, beacon age, # lock identity before and after close, and successor disposition. The separate -# state/.watch-triage.log remains exclusively the watcher's absorbed-wake debug -# log and is never written here. +# state/.watch-triage.log is the watcher's own bounded observational debug log, +# carries no lifecycle semantics, and is never written here. # # --restart: stop ONLY this FM_HOME's watcher (the pid recorded in THIS home's # state/.watch.lock) and own a fresh cycle, or attach if a verified live peer diff --git a/bin/fm-watch.sh b/bin/fm-watch.sh index 2f150af60d8..c2a431f0a5e 100755 --- a/bin/fm-watch.sh +++ b/bin/fm-watch.sh @@ -267,6 +267,40 @@ recorded_windows() { # below). FM_WEDGE_DEMAND_INSPECT_COUNT=${FM_WEDGE_DEMAND_INSPECT_COUNT:-3} +# Shadow mode records wedge settlements and logs a graded score beside the fixed +# timer, but never changes escalation. +wedge_shadow_settle() { # + local win=$1 task=$2 idle=$3 outcome=$4 + [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + "$SCRIPT_DIR/fm-wedge-score.py" settle --state "$STATE" \ + --window "$win" --task "$task" --idle-secs "$idle" --outcome "$outcome" \ + >/dev/null 2>&1 || true +} + +wedge_shadow_score() { # + local task=$1 idle=$2 score + [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + score=$("$SCRIPT_DIR/fm-wedge-score.py" score --state "$STATE" \ + --task "$task" --idle-secs "$idle" \ + --fixed-threshold "$STALE_ESCALATE_SECS" 2>/dev/null) || true + [ -n "$score" ] && triage_log "shadow wedge score: $score" || true +} + +wedge_shadow_resumed() { # + local win=$1 task=$2 since_file=$3 since idle + [ "${FM_WEDGE_SHADOW:-1}" = 1 ] || return 0 + command -v python3 >/dev/null 2>&1 || return 0 + [ -e "$since_file" ] || return 0 + since=$(cat "$since_file" 2>/dev/null || true) + case "$since" in + ''|*[!0-9]*) return 0 ;; + esac + idle=$(( $(date +%s) - since )) + wedge_shadow_settle "$win" "$task" "$idle" resumed || true +} + # Repeat-poll wedge-timer bookkeeping for an already-classified stale hash # absorbed as provably-working - repairs a missing/corrupt timer (self-heals a # watcher restart between recording the hash and recording the timer), or @@ -292,6 +326,8 @@ wedge_timer_check() { # key=${key//\//_} key=${key//./_} clear_pause_state "$win" + wedge_shadow_resumed "$win" "$(window_to_task "$win" "$STATE")" \ + "$STATE/.stale-since-$key" || true rm -f "$STATE/.stale-$key" "$STATE/.stale-since-$key" "$STATE/.wedge-escalations-$key" } @@ -1061,6 +1099,7 @@ EOF if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then wedge_timer_check "$w" "$ssf" "busy (no completed turn)" "$ewf" else + wedge_shadow_resumed "$w" "$task" "$ssf" || true rm -f "$ssf" "$ewf" fi if [ -e "$pf" ] && { [ "$n" -ge 2 ] || ! status_is_paused_or_captain_held "$(last_status_line "$STATE/$(window_to_task "$w" "$STATE").status")"; }; then @@ -1073,6 +1112,7 @@ EOF if [ "$busy_now" -eq 0 ] && busy_turn_over_age "$task"; then wedge_timer_check "$w" "$ssf" "busy (no completed turn)" "$ewf" else + wedge_shadow_resumed "$w" "$task" "$ssf" || true rm -f "$ssf" "$ewf" fi task=$(window_to_task "$w" "$STATE") diff --git a/bin/fm-wedge-score.py b/bin/fm-wedge-score.py new file mode 100755 index 00000000000..a3c0058743a --- /dev/null +++ b/bin/fm-wedge-score.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# Usage: fm-wedge-score.py settle|score --state [options]. +# JSONL schema: {"ts":"","window":"","task":"","lane":"","idle_secs":,"outcome":"resumed|escalated"}. +# This script is the single owner of the .wedge-settlements.jsonl schema, of the +# task-to-lane rule, and of the log's size bound. The log is watcher-owned +# bookkeeping and lives beside .watch-triage.log in the state directory the +# caller resolves. +# +# The graded score is a shadow comparator, not a calibrated one: its healthy +# sample is right-censored by the fixed timer it shadows (a pane idle past the +# threshold escalates instead of settling as resumed), so it never observes a +# legitimately long healthy tail. + +import argparse +import json +import math +import os +import sys +from datetime import datetime, timezone + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from argyle_gates import norm_cdf, norm_ppf + +SETTLEMENTS = ".wedge-settlements.jsonl" +MAX_BYTES = 262144 + + +def _lane_for_task(state, task): + if not task: + return "unknown" + meta = os.path.join(state, f"{task}.meta") + try: + with open(meta, encoding="utf-8") as handle: + for line in handle: + if line.startswith("project="): + return line.rstrip("\n").split("=", 1)[1] or "unknown" + except OSError: + pass + return "unknown" + + +def _trim(path): + if os.path.getsize(path) < MAX_BYTES: + return + kept, size = [], 0 + with open(path, encoding="utf-8") as handle: + for line in reversed(handle.readlines()): + size += len(line.encode("utf-8")) + if size > MAX_BYTES // 2: + break + kept.append(line) + kept.reverse() + tmp = f"{path}.tmp" + with open(tmp, "w", encoding="utf-8") as handle: + handle.writelines(kept) + os.replace(tmp, path) + + +def _settle(args): + try: + os.makedirs(args.state, exist_ok=True) + row = { + "ts": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"), + "window": args.window, + "task": args.task, + "lane": _lane_for_task(args.state, args.task), + "idle_secs": args.idle_secs, + "outcome": args.outcome, + } + path = os.path.join(args.state, SETTLEMENTS) + with open(path, "a", encoding="utf-8") as handle: + json.dump(row, handle, separators=(",", ":")) + handle.write("\n") + _trim(path) + except Exception as exc: + print(f"fm-wedge-score settle: {exc}", file=sys.stderr) + return 0 + + +def _read_rows(state, lane): + path = os.path.join(state, SETTLEMENTS) + rows = [] + try: + with open(path, encoding="utf-8") as handle: + for line in handle: + try: + row = json.loads(line) + except (TypeError, ValueError): + continue + if isinstance(row, dict) and row.get("lane") == lane: + rows.append(row) + except OSError: + pass + return rows + + +def _escalation_incidents(rows): + # A pane that stays wedged re-escalates every threshold, so only the first + # escalation of a window since its last resume counts as an incident. + unresolved = set() + count = 0 + for row in rows: + window = row.get("window", "") + if row.get("outcome") == "escalated": + if window not in unresolved: + unresolved.add(window) + count += 1 + elif row.get("outcome") == "resumed": + unresolved.discard(window) + return count + + +def _score(args): + lane = _lane_for_task(args.state, args.task) + rows = _read_rows(args.state, lane) + healthy = [ + float(row["idle_secs"]) + for row in rows + if row.get("outcome") == "resumed" + ] + n = len(healthy) + n_esc = _escalation_incidents(rows) + n_total = n + n_esc + fixed_flag = args.idle_secs >= args.fixed_threshold + if n < args.min_obs: + result = { + "lane": lane, + "n": n, + "score": None, + "graded_flag": None, + "fixed_flag": fixed_flag, + "reason": f"insufficient data: {n}<{args.min_obs}", + } + else: + logs = [math.log(max(value, 1.0)) for value in healthy] + mu = sum(logs) / n + variance = sum((value - mu) ** 2 for value in logs) / (n - 1) + sigma = max(math.sqrt(variance), 0.25) + z = (math.log(max(args.idle_secs, 1)) - mu) / sigma + base_rate = (n_esc + 1) / (n_total + 2) + score = z - norm_ppf(1 - base_rate) + p_tail = 1 - norm_cdf(z) + result = { + "lane": lane, + "n": n, + "n_total": n_total, + "base_rate": round(base_rate, 4), + "z": round(z, 4), + "p_tail": round(p_tail, 4), + "score": round(score, 4), + "graded_flag": score > 0, + "fixed_flag": fixed_flag, + "reason": "ok", + } + print(json.dumps(result, separators=(",", ":"))) + return 0 + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(dest="command", required=True) + + settle = subparsers.add_parser("settle") + settle.add_argument("--state", required=True) + settle.add_argument("--window", required=True) + settle.add_argument("--task", default="") + settle.add_argument("--idle-secs", required=True, type=int) + settle.add_argument("--outcome", choices=("resumed", "escalated"), required=True) + settle.set_defaults(handler=_settle) + + score = subparsers.add_parser("score") + score.add_argument("--state", required=True) + score.add_argument("--task", default="") + score.add_argument("--idle-secs", required=True, type=int) + score.add_argument("--fixed-threshold", required=True, type=int) + score.add_argument("--min-obs", default=8, type=int) + score.set_defaults(handler=_score) + + args = parser.parse_args() + return args.handler(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/docs/architecture.md b/docs/architecture.md index e56bacdc2ba..d5fc00b7434 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,7 +60,7 @@ At session start, `bin/fm-session-start.sh` emits exactly one primary-harness su That block owns the live wait shape for the running primary harness: Claude's Stop `asyncRewake` hook owns tokenless re-arm cycles, Grok uses background-notify cycles, Codex uses bounded foreground checkpoints, Pi and pi-signed use the same two tracked primary extensions, and OpenCode uses its TUI plugin. `bin/fm-watch-arm.sh` remains the verified arm wrapper for protocols that call it; it forks the watcher as a tracked child, verifies it is genuinely alive with a fresh liveness beacon, and prints an honest `started`, `attached`, or nonzero `FAILED` status. [`watcher-continuity.md`](watcher-continuity.md#arm-layer-cycle-contract) owns the arm layer's successor, terminal-delivery, and typed clean-close failure contract. -The arm layer records one bounded lifecycle row per observed cycle in `state/.watch-cycle-exits.log`; `state/.watch-triage.log` remains exclusively the absorbed-wake debug log. +The arm layer records one bounded lifecycle row per observed cycle in `state/.watch-cycle-exits.log`, which is the only watcher log with lifecycle semantics. Pi and OpenCode verify session-lock ownership and launch one singleton successor from their child-close handlers before delivering an actionable wake prompt, with bounded exponential retry for failed restoration. Claude's `bin/fm-claude-stop-autoarm.sh` hook fires on every Stop and, when the home is eligible and still needs supervision, claims one home-scoped cycle, foregrounds the arm wrapper, and translates actionable closes into exit-2 rewakes. It suppresses failed-looking closes when the same identity-matched watcher is healthy, retries genuine failures within a bound, and coordinates exhausted failure episodes with the Claude turn-end guard as documented in [`turnend-guard.md`](turnend-guard.md). diff --git a/docs/configuration.md b/docs/configuration.md index 1d1c121ed7a..f920d471d5a 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -573,7 +573,8 @@ FM_STALE_ESCALATE_SECS=240 # idle seconds before a provably-working stal FM_BUSY_TURN_MAX_SECS=3600 # maximum age of a busy pane's latest state/.turn-ended marker, or its state/.meta spawn record before any turn completes, before the same wedge escalation used for a provably-working non-busy stale takes over; inspection-only, never an automatic interrupt or restart FM_PAUSE_RESURFACE_SECS=3600 # seconds before an idle declared external wait re-surfaces for a recheck in the watcher or away-mode daemon FM_WEDGE_DEMAND_INSPECT_COUNT=3 # consecutive provably-working stale escalations on the same unchanged pane before demand-deep-inspection is added -FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's absorbed-wake debug log +FM_WEDGE_SHADOW=1 # set to 0 to stop recording wedge settlements in state/.wedge-settlements.jsonl and logging the shadow graded score beside the fixed timer; the score is observational only (its healthy sample is right-censored by FM_STALE_ESCALATE_SECS) and never changes escalation +FM_WATCH_TRIAGE_LOG_MAX_BYTES=262144 # size cap for the watcher's bounded observational debug log (absorbed wakes and other non-actionable notes) FM_FLEET_SYNC_BOOTSTRAP_TIMEOUT= # optional seconds allowed for bootstrap's best-effort clone refresh; unset/blank defaults to max(20, 5 + 3 * origin-backed-project-count) FM_FLEET_PRUNE=1 # set to 0 to skip pruning local branches whose upstream is gone FM_STALE_WORKTREE_LOCK_AGE_SECS=30 # min mtime age before fm-teardown.sh treats a leftover worktree git index.lock as provably stale diff --git a/docs/watcher-continuity.md b/docs/watcher-continuity.md index 2ae9a6b17bb..b73b8483b8a 100644 --- a/docs/watcher-continuity.md +++ b/docs/watcher-continuity.md @@ -53,7 +53,7 @@ Only a cycle with no matching delivery record emits `watcher: FAILED - cycle end The arm layer appends one tab-separated record per observed cycle to `state/.watch-cycle-exits.log`. Each record includes arm and watcher PIDs, start and end timestamps, exit code and signal, classified reason, beacon age, lock identity before and after close, and successor disposition. The file is size-capped through `FM_WATCH_CYCLE_LOG_MAX_BYTES` and `FM_WATCH_CYCLE_LOG_KEEP_LINES`. -`state/.watch-triage.log` remains only the watcher's bounded absorbed-wake debug log and carries no lifecycle semantics. +`state/.watch-triage.log` remains only the watcher's own bounded observational debug log, written for absorbed wakes and other non-actionable watcher notes, and carries no lifecycle semantics. The default 300-second grace is unchanged. Only the watcher process touches `state/.last-watcher-beat`; no helper process can make a wedged watcher appear healthy. diff --git a/tests/fm-pi-watch-extension.test.sh b/tests/fm-pi-watch-extension.test.sh index f8883194898..cee8f8dee68 100755 --- a/tests/fm-pi-watch-extension.test.sh +++ b/tests/fm-pi-watch-extension.test.sh @@ -417,7 +417,11 @@ trap 'exit 0' TERM INT while :; do sleep 0.02; done SH chmod +x "$repo/bin/fm-watch-arm.sh" - out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_PI_ARM_READY_TIMEOUT_MS=250 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node --input-type=module 2>&1 <<'EOF' + # The readiness budget bounds a successor that never reports ready, so it + # also bounds how long the spawned login-shell arm child has to record its + # row. Sized near cold `bash -l` start-up cost it retires an attempt before + # the script runs and silently loses an arm row, so keep it far above that. + out=$(PLUGIN="$plugin" FM_HOME="$home" FM_ROOT_OVERRIDE="$repo" FM_ARM_LOG="$log" FM_PI_ARM_READY_TIMEOUT_MS=2000 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node --input-type=module 2>&1 <<'EOF' import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; @@ -441,7 +445,7 @@ writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); const mod = await import(pathToFileURL(process.env.PLUGIN).href); mod.default(pi); await tool.execute("tool-call-hung-successor", {}, undefined, undefined, {}); -for (let i = 0; i < 500 && !prompt; i += 1) { +for (let i = 0; i < 2000 && !prompt; i += 1) { await new Promise((resolve) => setTimeout(resolve, 10)); } const rows = existsSync(process.env.FM_ARM_LOG) @@ -1579,7 +1583,11 @@ trap 'exit 0' TERM INT while :; do sleep 0.02; done SH chmod +x "$repo/bin/fm-watch-arm.sh" - out=$(PLUGIN="$plugin" WORKTREE="$repo" FM_HOME="$home" FM_ARM_LOG="$log" FM_OPENCODE_ARM_READY_TIMEOUT_MS=250 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node 2>&1 <<'EOF' + # The readiness budget bounds a successor that never reports ready, so it + # also bounds how long the spawned login-shell arm child has to record its + # row. Sized near cold `bash -l` start-up cost it retires an attempt before + # the script runs and silently loses an arm row, so keep it far above that. + out=$(PLUGIN="$plugin" WORKTREE="$repo" FM_HOME="$home" FM_ARM_LOG="$log" FM_OPENCODE_ARM_READY_TIMEOUT_MS=2000 FM_WATCH_REARM_RETRY_BASE_MS=5 FM_WATCH_REARM_RETRY_MAX_MS=10 FM_WATCH_REARM_RETRY_LIMIT=2 node 2>&1 <<'EOF' import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { pathToFileURL } from "node:url"; @@ -1603,7 +1611,7 @@ const hooks = await mod.FmPrimaryWatchArm({ }); writeFileSync(`${process.env.FM_HOME}/state/.lock`, `${process.pid}\n`); await hooks.event({ event: { type: "session.idle", properties: { sessionID: "session-test" } } }); -for (let i = 0; i < 500 && !prompt; i += 1) { +for (let i = 0; i < 2000 && !prompt; i += 1) { await new Promise((resolve) => setTimeout(resolve, 10)); } const rows = existsSync(process.env.FM_ARM_LOG) diff --git a/tests/fm-wedge-score.test.sh b/tests/fm-wedge-score.test.sh new file mode 100755 index 00000000000..541c606876d --- /dev/null +++ b/tests/fm-wedge-score.test.sh @@ -0,0 +1,228 @@ +#!/usr/bin/env bash +# Regression tests for the fixed-timer wedge settlement shadow score. +set -u + +# shellcheck source=tests/lib.sh +. "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +command -v python3 >/dev/null 2>&1 || { echo "skip: python3 not found"; exit 0; } + +SCORE="$ROOT/bin/fm-wedge-score.py" +TMP_ROOT=$(fm_test_tmproot fm-wedge-score) +LOG=.wedge-settlements.jsonl + +test_settle_records_project_lane() { + local state out + state="$TMP_ROOT/project/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + "$SCORE" settle --state "$state" --window sess:win --task task \ + --idle-secs 12 --outcome resumed + out=$(python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + row = json.loads(handle.readline()) +assert row["lane"] == "demo", row +assert row["idle_secs"] == 12, row +assert row["outcome"] == "resumed", row +assert row["window"] == "sess:win", row +print("ok") +PY +) + [ "$out" = ok ] || fail "settle row validation failed" + pass "settle appends a row with the project lane" +} + +test_score_requires_minimum_observations() { + local state out check + state="$TMP_ROOT/insufficient/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for idle in (40, 50, 60): + handle.write(json.dumps({"lane": "demo", "window": "s:w", + "idle_secs": idle, + "outcome": "resumed"}) + "\n") +PY + out=$("$SCORE" score --state "$state" --task task --idle-secs 100 \ + --fixed-threshold 240) + check=$(python3 - "$out" <<'PY' +import json +import sys +row = json.loads(sys.argv[1]) +assert row["score"] is None, row +assert row["graded_flag"] is None, row +assert row["reason"] == "insufficient data: 3<8", row +print("ok") +PY +) || fail "insufficient-data assertions failed for $out" + [ "$check" = ok ] || fail "insufficient-data validation failed for $out" + pass "score reports insufficient data below the minimum observations" +} + +test_score_flags_graded_and_fixed_thresholds() { + local state high low check + state="$TMP_ROOT/scored/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for idle in (40, 45, 50, 55, 60, 65, 70, 80, 90, 100): + handle.write(json.dumps({"lane": "demo", "window": "s:w", + "idle_secs": idle, + "outcome": "resumed"}) + "\n") + handle.write(json.dumps({"lane": "demo", "window": "s:w", + "idle_secs": 240, + "outcome": "escalated"}) + "\n") +PY + high=$("$SCORE" score --state "$state" --task task --idle-secs 600 \ + --fixed-threshold 240) + low=$("$SCORE" score --state "$state" --task task --idle-secs 50 \ + --fixed-threshold 240) + check=$(python3 - "$high" "$low" <<'PY' +import json +import sys +high, low = (json.loads(value) for value in sys.argv[1:]) +assert high["graded_flag"] is True, high +assert high["fixed_flag"] is True, high +assert high["score"] > 0, high +assert low["graded_flag"] is False, low +assert low["fixed_flag"] is False, low +print("ok") +PY +) || fail "graded/fixed assertions failed for high=$high low=$low" + [ "$check" = ok ] || fail "graded/fixed validation failed for high=$high low=$low" + pass "score separates graded and fixed threshold flags" +} + +# The escalation bound is FM_STALE_ESCALATE_SECS, not a constant, so the caller +# supplies it and fixed_flag reports what the fixed timer actually did. +test_fixed_flag_follows_the_supplied_threshold() { + local state out check + state="$TMP_ROOT/threshold/state" + mkdir -p "$state" + out=$("$SCORE" score --state "$state" --task missing --idle-secs 120 \ + --fixed-threshold 120) + check=$(python3 - "$out" <<'PY' +import json +import sys +row = json.loads(sys.argv[1]) +assert row["fixed_flag"] is True, row +print("ok") +PY +) || fail "fixed_flag did not follow the supplied threshold: $out" + [ "$check" = ok ] || fail "fixed_flag validation failed for $out" + pass "fixed_flag follows the caller's escalation threshold" +} + +# A pane that stays wedged re-escalates every threshold; counting those repeats +# as separate incidents would inflate the base rate and lower the graded bar. +test_repeated_escalations_count_as_one_incident() { + local state repeated distinct check + state="$TMP_ROOT/incidents/state" + mkdir -p "$state" + printf 'project=demo\n' > "$state/task.meta" + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for idle in (40, 45, 50, 55, 60, 65, 70, 80): + handle.write(json.dumps({"lane": "demo", "window": "s:healthy", + "idle_secs": idle, + "outcome": "resumed"}) + "\n") + for _ in range(6): + handle.write(json.dumps({"lane": "demo", "window": "s:wedged", + "idle_secs": 240, + "outcome": "escalated"}) + "\n") +PY + repeated=$("$SCORE" score --state "$state" --task task --idle-secs 300 \ + --fixed-threshold 240) + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "a", encoding="utf-8") as handle: + for window in ("s:a", "s:b", "s:c", "s:d", "s:e"): + handle.write(json.dumps({"lane": "demo", "window": window, + "idle_secs": 240, + "outcome": "escalated"}) + "\n") +PY + distinct=$("$SCORE" score --state "$state" --task task --idle-secs 300 \ + --fixed-threshold 240) + check=$(python3 - "$repeated" "$distinct" <<'PY' +import json +import sys +repeated, distinct = (json.loads(value) for value in sys.argv[1:]) +assert repeated["n_total"] == 9, repeated +assert distinct["n_total"] == 14, distinct +assert distinct["base_rate"] > repeated["base_rate"], (repeated, distinct) +print("ok") +PY +) || fail "incident counting failed for repeated=$repeated distinct=$distinct" + [ "$check" = ok ] || fail "incident validation failed for repeated=$repeated distinct=$distinct" + pass "repeated escalations of one window count as a single incident" +} + +test_settle_missing_meta_uses_unknown_lane() { + local state out + state="$TMP_ROOT/missing-meta/state" + mkdir -p "$(dirname "$state")" + "$SCORE" settle --state "$state" --window sess:win --task missing \ + --idle-secs 4 --outcome escalated + out=$(python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + print(json.loads(handle.readline())["lane"]) +PY +) + [ "$out" = unknown ] || fail "missing meta lane was '$out'" + pass "settle uses unknown lane when task metadata is absent" +} + +test_settlement_log_stays_bounded() { + local state before after check + state="$TMP_ROOT/bounded/state" + mkdir -p "$state" + python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], "w", encoding="utf-8") as handle: + for i in range(4000): + handle.write(json.dumps({"ts": "2026-01-01T00:00:00Z", "window": "s:w", + "task": "old", "lane": "demo", + "idle_secs": i, "outcome": "resumed"}) + "\n") +PY + before=$(wc -c < "$state/$LOG" | tr -d '[:space:]') + [ "$before" -gt 262144 ] || fail "fixture log was only $before bytes" + "$SCORE" settle --state "$state" --window sess:win --task newest \ + --idle-secs 7 --outcome escalated + after=$(wc -c < "$state/$LOG" | tr -d '[:space:]') + [ "$after" -lt "$before" ] || fail "settlement log was not trimmed: $after bytes" + check=$(python3 - "$state/$LOG" <<'PY' +import json +import sys +with open(sys.argv[1], encoding="utf-8") as handle: + rows = [json.loads(line) for line in handle if line.strip()] +assert rows, "trimming emptied the log" +assert rows[-1]["task"] == "newest", rows[-1] +assert rows[-1]["idle_secs"] == 7, rows[-1] +print("ok") +PY +) || fail "the newest settlement did not survive trimming" + [ "$check" = ok ] || fail "trimmed log validation failed" + pass "the settlement log is trimmed once it passes its size cap" +} + +test_settle_records_project_lane +test_score_requires_minimum_observations +test_score_flags_graded_and_fixed_thresholds +test_fixed_flag_follows_the_supplied_threshold +test_repeated_escalations_count_as_one_incident +test_settle_missing_meta_uses_unknown_lane +test_settlement_log_stays_bounded