From 1bdb6393d3400c9613ca74910c1f6a8d4bfd8af9 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 21:53:40 +0200 Subject: [PATCH 01/26] docs(spec): bounded capture suites Co-Authored-By: Claude Opus 5.5 --- ...026-10-16-bounded-capture-suites-design.md | 315 ++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md diff --git a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md new file mode 100644 index 0000000..9008c57 --- /dev/null +++ b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md @@ -0,0 +1,315 @@ +# Bounded capture suites: `captures.max_age` + `captures.max_cases` + +**Date:** 2026-10-16 · **Status:** approach A approved in chat, spec for maintainer review · **Repos:** evalshift-cli (behaviour + docs); evalshift-sdk, evalshift-action, evalshift-server, evalshift-client (nothing — follow-ups recorded) + +## Problem + +A team that ships captures to a bucket (`EVALSHIFT_CAPTURE_STORE`) and switches +`EVALSHIFT_CAPTURE=1` on in production for a couple of days at a time writes thousands of +objects per agent each time. The SDK's 200-newest-per-suite bound applies to local disk only +(`docs/sdk.md:106-120`); the bucket keeps everything a lifecycle rule does not expire. Today: + +1. **One sync promotes everything.** `capture sync` promotes every local capture that survives + the no-events / blocked / errored / duplicate filters (`cli/commands/capture.py:1213-1318`), and + the fetch downloads every object not already promoted (`captures/remote.py:297-335`). Nothing + bounds a suite, so every CI run replays thousands of rows on both models plus the judge. +2. **Suites only grow.** No code deletes a promoted case — `capture clean` unlinks capture files + and orphan sidecars only (`capture.py:830, 898`). Week 2's sync adds to week 1's. +3. **Stale cases are invisible.** A case's ground truth is what the production model did when the + capture was recorded, but neither `PromotedCase` nor `SuiteExample` records when that was or on + which model (`captures/models.py:53-93`). `created_at` is never parsed (a string tie-break only, + `captures/promote.py:1308`) and `model_id` is read only for pricing (`promote.py:552`). After a + production migration the old-model cases keep gating CI, and a team that only runs CI — which + reads `golden.jsonl` and never touches the bucket — never learns its suite is stale. +4. **An empty checkout re-promotes the whole bucket.** The promoted set is derived only from the + committed case files (`captures/reader.py:410-423`). With those gone, every object is downloaded + and promoted again, however old. + +## Goals + +- A suite is a **bounded, deterministic, route-balanced sample of recent production on the current + source model**, refreshed deliberately by `capture sync` + a reviewed PR. +- Staleness is **visible where CI-only teams look**: `doctor` and `compare` output. +- A config that does not set the new keys behaves exactly as 1.5.0. + +Non-goals: the CLI deleting from the bucket (it stays read-only on the store); CI syncing on its +own; freshness in `report.html` / the bundle / the PR comment (follow-up, cross-repo). + +## Decision + +### 1. Config + +```yaml +captures: + store: s3://acme-evals/support-agent # optional, unchanged + max_age: 14d # optional + max_cases: 200 # optional, per suite +``` + +- `max_age` — a duration `N` + `m|h|d` (the relative grammar `--since` already accepts), parsed to a + `timedelta`, must be > 0. A new `utils/duration.py::parse_duration(text) -> timedelta` owns the + grammar; `captures/remote.py::parse_since` is refactored onto it and keeps its `--since` error + text. The config field uses an `Annotated[timedelta, BeforeValidator(...)]`, so a typo fails as + `captures.max_age: expected a duration like 30m, 24h or 14d`. +- `max_cases` — `int`, `ge=1`. +- Both default to `None`, meaning today's behaviour: no eligibility filter, no cap, no retirement. +- **Setting either one turns bounded suites on** — the eligibility filters (§3), the plan (§5) and + the freshness findings (§7). `defaults.source_model` alone changes nothing: it is set in nearly + every config, and keying model filtering on it would switch the feature on for every existing + user. +- Both are valid without `store` — they govern how captures become suites, from disk or bucket. +- No per-suite override in v1: the managed `suites:` region is regenerated on every sync and only + `evaluators.trace_invariants` is carried forward (`capture.py:981-1005`). Follow-up. +- Nothing is added to `defaults` — the hosted `eval_config_hash` hashes `defaults` + (`hosted/bundle.py:258-290`), so a new field there would orphan every hosted baseline. The + `captures` block is not hashed. + +### 2. What a promoted case remembers + +`PromotedCase` gains two fields (it is `extra="forbid"`, see Compatibility): + +| Field | Value | +| --- | --- | +| `captured_at: str = ""` | the envelope's `created_at`, verbatim (the SDK writes ISO 8601 UTC) | +| `source_model: str = ""` | `model_id` of the capture's **first** `model_call` — the call whose input the replay re-sends, the same event that already supplies `toolset_ref` and `generation_config` (`promote.py:299-301, 457-461`) | + +`capture sync` and `capture promote` write both. `SuiteExample` is **not** changed, so +`golden.jsonl` stays byte-compatible with the (possibly older) CLI that CI runs. + +**Backfill.** When sync sees an existing case with an empty field whose capture is local, it fills +the field and rewrites the case file; nothing else on the case changes (`promoted_at` included). + +**Legacy cases.** No `captured_at` → its age is taken from `promoted_at`, an upper bound on capture +time: the case looks younger than it is, so it is retired later, never earlier. Neither timestamp +parseable → age unknown → never age-stale. No `source_model` → never model-stale. + +### 3. Staleness and eligibility + +One clock instant `now` per sync run (a module-level `_now()` so tests can pin it). + +- **Age-stale** (a kept case) / **too old** (a candidate capture): captured before `now − max_age`. + Only when `max_age` is set. +- **Model-stale** / **other model**: `source_model` is known and + `same_model(source_model, defaults.source_model)` is false. Only when bounded suites are on (§1) + and `defaults.source_model` is set. + +`same_model(a, b)` — new `models/matching.py`, no config import. True when either: + +1. the **bare names** are equal, where bare = lowercase, drop everything up to the last `/`, then + strip one trailing snapshot date `-YYYYMMDD` or `-YYYY-MM-DD`; or +2. `resolve_model(a).id == resolve_model(b).id` (registry aliases). + +| a | b | match | via | +| --- | --- | --- | --- | +| `claude-sonnet-5-20260301` | `claude-sonnet-5` | yes | bare | +| `anthropic/claude-sonnet-5` | `claude-sonnet-5` | yes | bare | +| `google/gemini-3.8-flash` | `gemini-3.8-flash` | yes | bare | +| `gpt-5-2026-08-07` | `openai/gpt-5` | yes | bare | +| `claude-sonnet-5` | `claude-sonnet-5-5` | **no** | — | + +**Guardrail.** Per suite: when at least one kept case or candidate has a known model and every one +that does would be model-stale, the model check is switched off for that suite and sync warns +`⚠ planner: none of 212 cases/captures were recorded on claude-sonnet-5-5 (seen: claude-sonnet-5-20260301 ×212); not filtering by model — check defaults.source_model`. +A wrong or unrecognised `source_model` can therefore never empty or freeze a suite. The same rule +holds mid-rollout: until at least one new-model capture exists, nothing is filtered. + +**Eligible candidate** = a local capture that is not promoted, survives today's filters (events, +blocked, errored, dedup — §6), and is neither too old nor on another model. + +### 4. Fetch is bounded by time, not count + +Effective fetch cutoff = `--since` when given, else `now − max_age` when set, else none — applied +to the object's `last_modified` exactly as `--since` is today (`remote.py:324-331`). Promoted ids +are still skipped. + +The fetch does **not** sample: a listing yields only key, `last_modified` and size +(`remote.py:91-103`; capture ids are random `cap_`), so conversation, route and model are +unknown until download — and a turn without its own messages list rebuilds its history from its +sibling captures on disk (`promote.py:1369-1399`), so a key-level sample would silently truncate +histories. Volume inside the window is governed by `EVALSHIFT_SAMPLE_RATE` in the agent. A hard +download cap is a follow-up. + +### 5. The per-suite plan (only when `max_cases` is set) + +New pure module `captures/selection.py` — no I/O, no clock: + +```python +def plan_suite( + kept: Sequence[KeptCase], + candidates: Sequence[CandidateGroup], + *, + max_cases: int, +) -> SuitePlan: ... +``` + +`KeptCase`: name, captured instant (or `None`), stale reason (`None | "age" | "model"`), reviewed +flag, route, content key. `CandidateGroup`: group key, its turns (capture ids, in turn order), +route, content keys. `SuitePlan`: groups to add, `(case name, reason)` to retire, counts. + +- **Unit = conversation group.** All candidate turns sharing a `conversation_id` within the suite; + a capture without one is its own group. A group is added whole or not at all; one larger than + the remaining slots is skipped and the next tried. +- **Route** of a case or group = the sorted set of tool names in its round-1 `expected_tools` + across its turns, `∅` when none. +- **Order is deterministic, no RNG:** within a route, groups are taken in `sha256(group key)` order. + Same inputs ⇒ same plan on any machine; rerunning sync with nothing new changes nothing. +- **Partition kept cases:** `P` = reviewed — any of the case's expected tool calls carries + `provenance: reviewed` (never retired), `F` = fresh, `S` = stale. +- **Fill:** `slots = max(0, max_cases − |P| − |F|)`. Repeatedly pick the route with the fewest cases + in `P ∪ F ∪ added` among routes that still have a group that fits (ties → route string), add its + next group, until slots or groups run out. +- **Retire (replacement only):** `excess = |P| + |F| + |added| + |S| − max_cases`. While + `excess > 0`, retire from `S` in this order: (a) a stale case whose content key equals an added + turn's (its own replacement), (b) model-stale, (c) age-stale oldest first; ties by name. A stale + case is thus only ever retired to make room — **a suite never shrinks because it went stale.** +- **Trim:** if `excess > 0` once `S` is exhausted (a legacy or over-cap suite), retire from `F`: the + most-represented route each time, oldest first within it. `P` is never trimmed; if `|P| > max_cases` + sync warns and keeps them. + +Without `max_cases`: every eligible candidate is added (today's behaviour, minus too-old / +other-model captures when those filters are set), and nothing is retired. + +### 6. Sync integration + +The per-suite loop (`capture.py:1171-1321`) becomes **collect → plan → apply**: + +1. Build examples as today with `build_conversation_examples` over every local record of the suite, + promoted ones included, so history reconstruction keeps its siblings. +2. Classify each built record: already promoted (kept) / no events, blocked, errored (counted as + today) / too old / other model / eligible. +3. Dedup: seed from **`P ∪ F` only**, not `S`, so a question re-asked on the new model can replace + its stale twin (rule 5a); within-run first-wins as today. The final suite holds no duplicate + content keys unless `--keep-duplicates`. +4. `plan_suite`. +5. Apply: write added cases (with the §2 fields), unlink retired case files (new + `promote.retire_promoted_case`), backfill, then `rebuild_golden_jsonl` and + `derive_suite_evaluators` as today. `--repair` runs as today; the plan sees the post-repair set. + +**Which suites are planned:** every suite with local captures (as today) plus, when `max_cases` is +set, every `/suites//` holding case files, so an over-cap suite is trimmed even with no +new captures. `--suite` narrows both. The "no captures found" early return applies only when no +limits are set. + +**Config:** sync now loads `evalshift.yaml` even under `--offline` to read `captures.*` and +`defaults.source_model`. Missing or invalid config ⇒ no limits (today's behaviour), with the existing +warning when the text has a top-level `captures:` key (`capture.py:373-380`). + +**Flags:** no new flags — the limits are durable settings and live in YAML. `--since` keeps +overriding the fetch window; `--force`, `--keep-duplicates`, `--allow-errored`, `--repair`, `--print` +keep their meaning. + +**Output:** the existing summary line is unchanged. When limits are set, one line per planned suite: + +``` + planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model +``` + + +### 7. Freshness signals for CI-only teams + +New `captures/freshness.py::suite_freshness(suite_dir, *, max_age, max_cases, source_model, now) -> list[FreshnessFinding]`, +reading the case files beside `golden.jsonl` (they are committed with it). Computed only when +bounded suites are on (§1), and only for `source: captured` suites. Best-effort: unreadable +case files become one finding, never an exception. Findings: + +- `stale_age` — the newest case is older than `max_age` (`planner: newest case captured 23d ago (max_age 14d); refresh with evalshift capture sync`). +- `stale_model` — `k of n cases recorded on claude-sonnet-5; defaults.source_model is claude-sonnet-5-5`. +- `over_cap` — `planner: 2,940 cases, over captures.max_cases 200`. + +**doctor:** one `suite freshness` row per named captured suite with case files — `ok` +(`planner: 200 cases, captured 2–12 days ago`) or `warn` per finding — plus a `captures limits` +`warn` row when `store` is set and `max_cases` is not (`suites grow with every sync; set captures.max_cases and captures.max_age`). Never `fail`. + +**compare:** compare discards doctor `warn` rows (`compare.py:551-552`), so it calls +`suite_freshness` itself for the suite it runs and prints each finding as a `⚠` line above the +verdict, beside the tool-free warnings (`compare.py:716-719`). That is the CI log. + +### 8. `init` + +New projects get the limits active (the scaffold currently writes no `captures:` block, +`cli/commands/init.py:111-173`): + +```yaml +# How captures become suites. Each suite is a capped, route-balanced sample of +# production from the last max_age, on defaults.source_model; `capture sync` +# replaces stale cases as fresh captures arrive. Set store: if the SDK ships +# captures to a bucket (EVALSHIFT_CAPTURE_STORE). +captures: + max_age: 14d + max_cases: 200 + # store: s3:/// +``` + +Existing configs are not rewritten; doctor's `captures limits` row nudges them. + +### 9. Bucket retention (docs only) + +`docs/configuration.md#captures` and `docs/sdk.md` recommend a lifecycle rule that expires capture +objects after a period ≥ `max_age` (e.g. 30 days for `max_age: 14d`) — it bounds storage and how +long production user data sits in the bucket — with an `aws s3api put-bucket-lifecycle-configuration` +example and one-line GCS / Azure pointers. The CLI stays read-only on the bucket. + +## Trade-offs recorded + +- **Coverage-weighted, not traffic-weighted.** Route balancing means rate budgets such as + `max_overall_regression_rate` are measured over a balanced sample. Chosen because a rare tool path + regressing is what a migration gate exists to catch; a traffic-weighted 200-case sample gives a + 1%-traffic route about two cases. +- **Replacement-only retirement** means a suite can stay stale indefinitely if production stops + capturing. doctor and compare make that visible rather than emptying the suite. +- **Time-bounded fetch** downloads everything inside `max_age`, not just what the plan uses; correct + for conversations, bounded by the window and the SDK sample rate. + +## Not in scope (recorded follow-ups) + +- Freshness in `report.html`, the bundle, the hosted run page and the PR comment (bundle field; + evalshift-server + evalshift-action). +- Per-suite `max_cases` / `max_age` (carry-forward in the managed region). +- `capture sync --dry-run` (retirement deletes committed files; git restores them). +- A hard download cap inside the window. +- A traffic-proportional sampling mode. +- `code_version`-based staleness (opaque string, no ordering). +- Sweeping, or even reporting, the toolset sidecars a retirement leaves unreferenced + (`capture clean` already sweeps them). + +## Docs that change + +- `DOCS.md`: Capturing from production (`captures:` example + sync steps), Top-level fields + (`captures` row), How it works (doctor bullet), Command reference (sync, doctor, compare). +- `llms-full.txt`: the same sections plus the `captures:` schema block. +- `docs/configuration.md`: `## captures` field table, retention paragraph, Capture lifecycle. +- `docs/sdk.md`: §2 bucket-retention note, §3 promote. +- `docs/conversations.md`: conversations are sampled and kept whole. +- `CHANGELOG.md` `## [Unreleased]`: Added (`captures.max_age`, `captures.max_cases`, freshness + rows/lines, case fields), Changed (sync loads config under `--offline`; `init` scaffolds limits). + +## Compatibility + +- **Opt-in.** Without the new keys, sync promotes and keeps exactly what 1.5.0 does. Two + differences remain: case files gain `captured_at` / `source_model`, and sync loads the config + under `--offline`. +- **New config keys** are rejected by older CLIs (`extra="forbid"`): reader ≥ writer, bump the CI + action's `evalshift-version` — the same as `captures.store` in 1.3.0. +- **New `PromotedCase` fields**: an older CLI's `capture` commands fail closed on a new case file + (`_iter_promoted_cases` raises; the fetch skip list degrades to empty, `capture.py:182-190`). + Everyone who runs `capture` commands on the repo needs this version or later. CI is unaffected — + it reads `golden.jsonl` only, and that format does not change. +- `compute_config_hash` covers the whole config, so new defaulted fields change it for every config + and a `--resume` across the upgrade reports a mismatch (as in 1.3.0). The hosted + `eval_config_hash` does not hash `captures` and does not move. +- No SDK, action, server or client change. + +## Testing + +- **Unit:** `parse_duration`; `CapturesConfig` fields (valid, invalid, dotted error location); + `same_model` table; `plan_suite` properties — deterministic, idempotent, cap respected, + replacement-only, reviewed pinned, twin preferred, groups whole, route balance, trim order; + `suite_freshness` findings incl. legacy cases; promote writes the new fields; backfill; + doctor rows; compare lines; init scaffold (plus the existing docs-match test). +- **Helpers:** `fake_remote_store.capture_payload` and `test_cli_capture._write_capture` gain a + `model_id` parameter (both hardcode `"m"` today). +- **Integration** (`tests/integration/test_capture_lifecycle.py`), clock pinned: + week 1 sync caps at `max_cases`; week 2 inside `max_age` changes nothing; clock past `max_age` + with new captures replaces stale cases; `defaults.source_model` changed replaces model-stale + cases; a fresh checkout with committed suites fetches only new captures; a wrong `source_model` + trips the guardrail and filters nothing. From e75bc3b850e2632669dd748fa5e0d4e3460a248e Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 21:53:40 +0200 Subject: [PATCH 02/26] docs(plan): bounded capture suites Co-Authored-By: Claude Opus 5.5 --- .../2026-10-16-bounded-capture-suites.md | 3082 +++++++++++++++++ 1 file changed, 3082 insertions(+) create mode 100644 docs/superpowers/plans/2026-10-16-bounded-capture-suites.md diff --git a/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md b/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md new file mode 100644 index 0000000..09784f5 --- /dev/null +++ b/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md @@ -0,0 +1,3082 @@ +# Bounded Capture Suites Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `captures.max_age` + `captures.max_cases` turn each captured suite into a bounded, deterministic, route-balanced sample of recent production on the current source model, refreshed by `capture sync`, with staleness visible in `doctor` and `compare`. + +**Architecture:** Two pure modules carry the logic — `captures/staleness.py` (what makes a case stale: age vs `max_age`, model vs `defaults.source_model` through `models/matching.py`) and `captures/selection.py` (`plan_suite`: fill free slots route-balanced in `sha256` order, retire stale cases only to make room, trim over-cap suites). `capture sync` becomes collect → plan → apply when `max_cases` is set; with neither limit set it is byte-for-byte today's sync. Promoted case files gain `captured_at` / `source_model`; `captures/freshness.py` reads them for the doctor rows and the compare lines. + +**Tech Stack:** Python 3.11+, pydantic v2, Typer + Rich, pytest, uv. + +**Spec:** `docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md` — read it first; this plan argues from it. + +## Global Constraints + +- Python floor 3.11 (no 3.12+ syntax). Every module starts with `from __future__ import annotations`; public functions/classes get Google-style docstrings; `mypy --strict src/evalshift_cli` stays clean. +- **Opt-in rule (spec §1):** every new sync behaviour — the too-old / other-model filters, planning, retirement — and every freshness finding applies **only when `captures.max_age` or `captures.max_cases` is set**. `defaults.source_model` alone changes nothing. Writing and backfilling `captured_at` / `source_model` on case files happens always. +- Config key names, verbatim: `captures.max_age` (duration `N` + `m|h|d`, > 0), `captures.max_cases` (int ≥ 1). Both default `None`. Nothing is added to `defaults` (the hosted `eval_config_hash` hashes it). +- `SuiteExample` / `golden.jsonl` do **not** change. Only `PromotedCase` gains fields. +- The CLI never writes to or deletes from a bucket. +- No new CLI flags (durable settings live in `evalshift.yaml`; `--since` stays the one-off fetch override). +- User-facing strings, verbatim: + - config error: `expected a duration like 30m, 24h or 14d` + - fetch skip label: `older than captures.max_age (14d)` (the `14d` is `format_duration(max_age)`) + - per-suite line: ` {suite}: {total} cases — kept {kept}, retired {n}[ ({reasons})], added {added} of {eligible} eligible[; not eligible: {reasons}]`; with `max_age` only: ` {suite}: not eligible: {reasons}`; reasons are `N older than 14d`, `N on another model`, `N over max_cases`, zero counts left out + - guardrail: `{suite}: none of {n} case(s) and capture(s) were recorded on {source_model} (seen: {models}); not filtering by model — check defaults.source_model` + - backfill clause: `, backfilled capture time and model on {n} case(s)` + - limits skipped (`--offline` only): `could not read {path} ({reason}); syncing without captures.max_age / captures.max_cases.` + - doctor rows `suite freshness` / `captures limits`; finding texts are in Task 9. +- Gate ("green" = all four, as CI): `uv run ruff check . && uv run ruff format --check . && uv run mypy --strict src/evalshift_cli && uv run pytest -q` (coverage floor 90). `make ci` is the exact CI mirror. +- Work on branch `feat/bounded-capture-suites` in a worktree (superpowers:using-git-worktrees). Before every commit confirm the branch is not `main`. Conventional Commits, each message ending with `Co-Authored-By: Claude Opus 5.5 `. Never `--no-verify`; if the pre-commit hook cannot find its Python in the worktree, commit with `uv run --with pre-commit git commit …`. +- `docs/superpowers/` is gitignored: add the spec and this plan with `git add -f` (Task 11). +- Subagents implementing or reviewing tasks run with `model: opus`. + +## Review Focus + +- Re-running `capture sync` with nothing new must not touch a single file — Task 6 `test_planning_twice_changes_nothing`, Task 8 `test_syncing_again_with_nothing_new_changes_nothing`. +- A `defaults.source_model` that no capture ran on (typo, alias, mid-rollout) must neither empty nor freeze a suite — Task 8 `test_a_source_model_no_capture_ran_on_filters_nothing`, Task 10 `test_a_wrong_source_model_trips_the_guardrail`. +- A conversation larger than the free slots is never split — Task 6 `test_a_group_is_added_whole_or_not_at_all`, Task 8 `test_a_conversation_enters_whole_or_not_at_all`. +- `capture sync --suite alpha` must not retire or trim any other suite — Task 8 `test_suite_flag_leaves_other_suites_alone`. +- Capture timestamps in `Z`, a non-UTC offset, naive, or garbage — Task 5 `test_parse_instant_normalises_to_utc`, `test_parse_instant_is_none_for_anything_unparseable`. + +--- + +### Task 1: Duration grammar and the two config fields + +**Files:** +- Create: `src/evalshift_cli/utils/duration.py` +- Modify: `src/evalshift_cli/captures/remote.py` (`parse_since`, drop `_DURATION`) +- Modify: `src/evalshift_cli/config/models.py` (`CapturesConfig`, imports) +- Test: `tests/unit/test_utils_duration.py` (new), `tests/unit/test_config_captures.py` + +**Interfaces:** +- Produces: `utils.duration.DURATION_FORMS: str = "30m, 24h or 14d"`, `parse_duration(text: str) -> timedelta`, `format_duration(delta: timedelta) -> str`; `CapturesConfig.max_age: timedelta | None`, `CapturesConfig.max_cases: int | None`. + +- [ ] **Step 1: Write the failing duration tests** + +Create `tests/unit/test_utils_duration.py`: + +```python +"""The duration grammar shared by ``--since`` and ``captures.max_age``.""" + +from __future__ import annotations + +from datetime import timedelta + +import pytest + +from evalshift_cli.utils.duration import format_duration, parse_duration + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("30m", timedelta(minutes=30)), + ("24h", timedelta(hours=24)), + ("14d", timedelta(days=14)), + (" 7d ", timedelta(days=7)), + ("0d", timedelta(0)), + ], +) +def test_parse_duration_accepts_minutes_hours_days(text: str, expected: timedelta) -> None: + assert parse_duration(text) == expected + + +@pytest.mark.parametrize("text", ["", "14", "2w", "1.5d", "-1d", "d", "14 d", "P14D"]) +def test_parse_duration_rejects_anything_else(text: str) -> None: + with pytest.raises(ValueError, match="expected a duration like 30m, 24h or 14d"): + parse_duration(text) + + +def test_parse_duration_overflow_is_a_value_error() -> None: + with pytest.raises(ValueError, match="too large"): + parse_duration("9999999999999d") + + +@pytest.mark.parametrize( + ("delta", "text"), + [(timedelta(days=14), "14d"), (timedelta(hours=36), "36h"), (timedelta(minutes=90), "90m")], +) +def test_format_duration_uses_the_largest_whole_unit(delta: timedelta, text: str) -> None: + assert format_duration(delta) == text +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_utils_duration.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'evalshift_cli.utils.duration'` + +- [ ] **Step 3: Implement `utils/duration.py`** + +```python +"""The duration grammar shared by ``--since`` and ``captures.max_age``: ``N`` + ``m|h|d``.""" + +from __future__ import annotations + +import re +from datetime import timedelta + +#: How an error names the accepted forms. +DURATION_FORMS = "30m, 24h or 14d" + +_DURATION = re.compile(r"(\d+)([mhd])") +_UNITS = {"m": timedelta(minutes=1), "h": timedelta(hours=1), "d": timedelta(days=1)} + + +def parse_duration(text: str) -> timedelta: + """Parse ``30m`` / ``24h`` / ``14d`` (surrounding whitespace allowed) into a timedelta. + + Raises: + ValueError: when ``text`` is not that form, or is too large for a timedelta. + """ + match = _DURATION.fullmatch(text.strip()) + if match is None: + raise ValueError(f"expected a duration like {DURATION_FORMS}, got {text!r}") + try: + return _UNITS[match.group(2)] * int(match.group(1)) + except OverflowError as exc: + raise ValueError(f"duration {text!r} is too large") from exc + + +def format_duration(delta: timedelta) -> str: + """Render ``delta`` in the largest whole unit :func:`parse_duration` reads (``14d``, ``36h``).""" + seconds = int(delta.total_seconds()) + for suffix, unit in (("d", 86400), ("h", 3600), ("m", 60)): + if seconds % unit == 0: + return f"{seconds // unit}{suffix}" + return f"{seconds}s" +``` + +- [ ] **Step 4: Move `parse_since` onto it** + +In `src/evalshift_cli/captures/remote.py` delete the `_DURATION = re.compile(r"(\d+)([mhd])")` line, import `from evalshift_cli.utils.duration import parse_duration`, and replace the body of `parse_since` (keep its signature and docstring): + +```python + reference = now if now is not None else datetime.now(UTC) + cleaned = text.strip() + try: + delta: timedelta | None = parse_duration(cleaned) + except ValueError: + delta = None + try: + if delta is not None: + return reference - delta + parsed = datetime.fromisoformat(cleaned) + except (ValueError, OverflowError) as exc: # OverflowError: a duration reaching before year 1 + raise ValueError( + f"--since expects a duration like 30m, 24h, 7d or an ISO date/datetime, got {text!r}" + ) from exc + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=UTC) +``` + +If `re` / `timedelta` imports become unused or newly needed, fix them (`ruff check` says which). + +- [ ] **Step 5: Run the duration and `--since` tests** + +Run: `uv run pytest tests/unit/test_utils_duration.py tests/unit/test_captures_remote.py -q` +Expected: PASS (the existing `parse_since` tests at `test_captures_remote.py:333-362` pin the unchanged behaviour). + +- [ ] **Step 6: Write the failing config tests** + +Append to `tests/unit/test_config_captures.py` (add `from datetime import timedelta` to the imports): + +```python +def test_limits_default_to_off() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT]) + assert cfg.captures.max_age is None + assert cfg.captures.max_cases is None + + +def test_limits_parse_without_a_store() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": "14d", "max_cases": 200}) + assert cfg.captures.max_age == timedelta(days=14) + assert cfg.captures.max_cases == 200 + assert cfg.captures.store is None + + +@pytest.mark.parametrize("value", ["2w", "14", 14, "0d", "P14D"]) +def test_max_age_rejects_anything_but_a_positive_duration(value: object) -> None: + with pytest.raises(ValidationError, match="max_age"): + EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": value}) + + +@pytest.mark.parametrize("value", [0, -5]) +def test_max_cases_must_be_positive(value: int) -> None: + with pytest.raises(ValidationError, match="max_cases"): + EvalShiftConfig(prompts=[_PROMPT], captures={"max_cases": value}) + + +def test_loader_reports_max_age_error_at_its_location(tmp_path: Path) -> None: + path = tmp_path / "evalshift.yaml" + path.write_text( + "prompts:\n - id: p\n detection: manual\n content: hi\n" + "captures:\n max_age: 2w\n", + encoding="utf-8", + ) + with pytest.raises(ConfigError) as info: + load_config(path) + [detail] = info.value.details + assert detail.location == "captures.max_age" + assert "30m, 24h or 14d" in detail.message +``` + +- [ ] **Step 7: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_config_captures.py -q` +Expected: FAIL — `Extra inputs are not permitted` for `max_age` / `max_cases`. + +- [ ] **Step 8: Add the fields** + +In `src/evalshift_cli/config/models.py`: add `from datetime import timedelta`, add `BeforeValidator` to the pydantic import, and `from evalshift_cli.utils.duration import DURATION_FORMS, parse_duration`. Above `class CapturesConfig` add: + +```python +def _duration_from_yaml(value: object) -> object: + """``captures.max_age`` is written ``14d``; a bare number would silently mean seconds.""" + if value is None or isinstance(value, timedelta): + return value + if not isinstance(value, str): + raise ValueError(f"expected a duration like {DURATION_FORMS}") + return parse_duration(value) +``` + +In `CapturesConfig`, extend the docstring's `Attributes:` with + +``` + max_age: Captures recorded longer ago than this (``30m``, ``24h``, ``14d``) never enter a + suite, and promoted cases that old are stale: ``capture sync`` replaces them as fresh + captures arrive. Also the fetch window. ``None`` (the default): no age limit. + max_cases: The most cases ``capture sync`` keeps in each captured suite. ``None`` (the + default): suites grow with every sync. Setting either limit turns bounded suites on. +``` + +and add the fields and validator under `store`: + +```python + max_age: Annotated[timedelta | None, BeforeValidator(_duration_from_yaml)] = None + max_cases: int | None = Field(default=None, ge=1) + + @field_validator("max_age") + @classmethod + def _positive_max_age(cls, value: timedelta | None) -> timedelta | None: + if value is not None and value <= timedelta(0): + raise ValueError(f"must be greater than 0 (expected a duration like {DURATION_FORMS})") + return value +``` + +- [ ] **Step 9: Run the config tests, then the gate's static checks** + +Run: `uv run pytest tests/unit/test_config_captures.py tests/unit/test_config_models.py tests/unit/test_config_loader.py -q && uv run mypy --strict src/evalshift_cli && uv run ruff check . && uv run ruff format --check .` +Expected: PASS / clean. + +- [ ] **Step 10: Commit** + +```bash +git add src/evalshift_cli/utils/duration.py src/evalshift_cli/captures/remote.py src/evalshift_cli/config/models.py tests/unit/test_utils_duration.py tests/unit/test_config_captures.py +git commit -m "feat(config): captures.max_age and captures.max_cases + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 2: `init` scaffolds the limits + +**Files:** +- Modify: `src/evalshift_cli/cli/commands/init.py` (new `CAPTURES_SCAFFOLD`, `render_minimal_config`) +- Modify: `tests/unit/test_cli_capture.py` (`_write_min_config`), `tests/unit/test_cli_capture_remote.py` (`_write_config`), `tests/unit/test_cli_capture_repair.py` (both `render_minimal_config` calls) +- Test: `tests/unit/test_init.py` + +**Interfaces:** +- Consumes: `CapturesConfig.max_age` / `max_cases` (Task 1). +- Produces: `cli.commands.init.CAPTURES_SCAFFOLD: Final[str]` — the exact block, ending in `\n`; tests strip it with `.replace(CAPTURES_SCAFFOLD + "\n", "")` to get today's unbounded sync. + +- [ ] **Step 1: Write the failing test** + +Append to `tests/unit/test_init.py` (import `re`, `timedelta`, `yaml`, `CAPTURES_SCAFFOLD` from `evalshift_cli.cli.commands.init`, `SUITES_MARKER_BEGIN` from `evalshift_cli.cli.commands._suites`, `EvalShiftConfig` from `evalshift_cli.config.models` — skip any already imported): + +```python +def test_scaffold_bounds_capture_suites() -> None: + text = render_minimal_config(profile="model-upgrade") + assert CAPTURES_SCAFFOLD in text + cfg = EvalShiftConfig.model_validate(yaml.safe_load(text)) + assert cfg.captures.max_age == timedelta(days=14) + assert cfg.captures.max_cases == 200 + assert cfg.captures.store is None + # The managed suites region stays last: capture sync rewrites only the tail. + block = re.search(r"(?m)^captures:$", text) + assert block is not None + assert block.start() < text.index(SUITES_MARKER_BEGIN) +``` + +- [ ] **Step 2: Run it to see it fail** + +Run: `uv run pytest tests/unit/test_init.py::test_scaffold_bounds_capture_suites -q` +Expected: FAIL — `ImportError: cannot import name 'CAPTURES_SCAFFOLD'` + +- [ ] **Step 3: Implement** + +In `src/evalshift_cli/cli/commands/init.py`, after `_MINIMAL_YAML_BODY` add: + +```python +#: The scaffold's ``captures:`` block, between the migration policy and the suites region. +#: Public so a test that needs today's unbounded sync can strip it from the scaffold. +CAPTURES_SCAFFOLD: Final = """\ +# How captures become suites. Each suite stays a capped sample of production +# from the last max_age, recorded on defaults.source_model: `capture sync` +# swaps stale cases for fresh captures as they arrive. Does the SDK ship +# captures to a bucket (EVALSHIFT_CAPTURE_STORE on the agent)? Uncomment +# store: and name the same bucket. +captures: + max_age: 14d + max_cases: 200 + # store: s3:/// +""" +``` + +and change the last line of `render_minimal_config` to + +```python + return ( + header + + body.rstrip() + + "\n\n" + + policy.rstrip() + + "\n\n" + + CAPTURES_SCAFFOLD + + "\n" + + suites + + "\n" + ) +``` + +- [ ] **Step 4: Keep the existing sync tests on unbounded sync** + +In `tests/unit/test_cli_capture.py` import `CAPTURES_SCAFFOLD` beside `render_minimal_config` and change `_write_min_config` to: + +```python +def _write_min_config(path: Path) -> None: + """Write a minimal init-style config (with the managed suites markers). + + Without the scaffold's capture limits: these tests pin unbounded sync, and their + fixtures' June 2026 captures are older than the scaffold's 14d max_age. + """ + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + path.write_text(text, encoding="utf-8") +``` + +Apply the same `.replace(CAPTURES_SCAFFOLD + "\n", "")` to the `render_minimal_config(...)` result in `tests/unit/test_cli_capture_remote.py::_write_config` and to both `render_minimal_config(...)` calls in `tests/unit/test_cli_capture_repair.py` (importing `CAPTURES_SCAFFOLD` in each). `_write_config` then appends its own `captures:` block as before. + +- [ ] **Step 5: Run init and every capture CLI test** + +Run: `uv run pytest tests/unit/test_init.py tests/unit/test_cli_capture.py tests/unit/test_cli_capture_remote.py tests/unit/test_cli_capture_repair.py tests/unit/test_docs_currency.py -q` +Expected: PASS. If a `test_init.py` test pins the exact scaffold text, update it to include `CAPTURES_SCAFFOLD`. + +- [ ] **Step 6: Commit** + +```bash +git add src/evalshift_cli/cli/commands/init.py tests/unit/test_init.py tests/unit/test_cli_capture.py tests/unit/test_cli_capture_remote.py tests/unit/test_cli_capture_repair.py +git commit -m "feat(init): scaffold captures.max_age and captures.max_cases + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 3: Model matching + +**Files:** +- Create: `src/evalshift_cli/models/matching.py` +- Test: `tests/unit/test_model_matching.py` (new) + +**Interfaces:** +- Consumes: `models.registry.resolve_model(id_or_alias: str) -> ModelMetadata` (existing). +- Produces: `bare_model_name(model_id: str) -> str`, `same_model(a: str, b: str) -> bool`, `model_filter_applies(recorded: Iterable[str], source_model: str | None) -> bool`, `describe_models(recorded: Iterable[str], *, limit: int = 3) -> str`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/unit/test_model_matching.py`: + +```python +"""Matching a capture's recorded model id against ``defaults.source_model``.""" + +from __future__ import annotations + +import pytest + +from evalshift_cli.models.matching import describe_models, model_filter_applies, same_model + + +@pytest.mark.parametrize( + ("a", "b"), + [ + ("claude-sonnet-5-20260301", "claude-sonnet-5"), + ("anthropic/claude-sonnet-5", "claude-sonnet-5"), + ("google/gemini-3.8-flash", "gemini-3.8-flash"), + ("gpt-5-2026-08-07", "openai/gpt-5"), + ("Claude-Sonnet-5", "claude-sonnet-5"), + ], +) +def test_same_model_matches_prefix_case_and_snapshot_variants(a: str, b: str) -> None: + assert same_model(a, b) + assert same_model(b, a) + + +@pytest.mark.parametrize( + ("a", "b"), + [ + ("claude-sonnet-5", "claude-sonnet-5-5"), + ("gpt-5", "gpt-5-mini"), + ("", "claude-sonnet-5"), + ("m", "claude-sonnet-5"), + ], +) +def test_same_model_keeps_distinct_models_apart(a: str, b: str) -> None: + assert not same_model(a, b) + + +def test_filter_applies_when_any_recorded_model_matches() -> None: + assert model_filter_applies(["claude-sonnet-5-20260301", "gpt-5"], "claude-sonnet-5") + + +def test_filter_is_off_without_a_source_model() -> None: + assert not model_filter_applies(["claude-sonnet-5"], None) + + +def test_filter_is_off_when_nothing_recorded_a_model() -> None: + assert not model_filter_applies(["", ""], "claude-sonnet-5") + + +def test_guardrail_filter_is_off_when_no_recorded_model_matches() -> None: + assert not model_filter_applies(["claude-sonnet-5"] * 3, "claude-sonnet-5-5") + + +def test_describe_models_counts_the_most_recorded_first() -> None: + recorded = ["a"] * 3 + ["b"] + ["c"] * 2 + ["d"] + [""] + assert describe_models(recorded) == "a ×3, c ×2, b ×1, +1 more" +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_model_matching.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'evalshift_cli.models.matching'` + +- [ ] **Step 3: Implement `models/matching.py`** + +```python +"""Whether two model ids name the same model, for capture-suite staleness. + +A capture records whatever id the provider echoed back -- often a dated snapshot +(``claude-sonnet-5-20260301``) or a provider-prefixed id -- while ``defaults.source_model`` +is usually the bare alias. Matching errs toward "same": a false match keeps a stale case a +little longer, a false mismatch would retire good ones. +""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable + +from evalshift_cli.models.registry import resolve_model + +_SNAPSHOT_SUFFIX = re.compile(r"-(?:\d{8}|\d{4}-\d{2}-\d{2})$") + + +def bare_model_name(model_id: str) -> str: + """``model_id`` lower-cased, without its provider prefix or a trailing snapshot date.""" + name = model_id.strip().lower().rsplit("/", 1)[-1] + return _SNAPSHOT_SUFFIX.sub("", name) + + +def same_model(a: str, b: str) -> bool: + """True when ``a`` and ``b`` share a bare name or resolve to one registry id.""" + if not a.strip() or not b.strip(): + return False + if bare_model_name(a) == bare_model_name(b): + return True + return resolve_model(a).id == resolve_model(b).id + + +def model_filter_applies(recorded: Iterable[str], source_model: str | None) -> bool: + """Whether filtering one suite by ``source_model`` is safe (the spec §3 guardrail). + + False without a source model, when nothing recorded a model, and when not one recorded + model matches it -- a wrong or unrecognised ``defaults.source_model``, or a migration whose + first new-model capture has not arrived yet, must never empty or freeze a suite. + """ + if not source_model: + return False + return any(same_model(model, source_model) for model in recorded if model) + + +def describe_models(recorded: Iterable[str], *, limit: int = 3) -> str: + """``a ×3, c ×2, b ×1, +1 more``: the most-recorded ids first, at most ``limit`` of them.""" + counts = Counter(model for model in recorded if model) + shown = ", ".join(f"{model} ×{n}" for model, n in counts.most_common(limit)) + rest = len(counts) - limit + return f"{shown}, +{rest} more" if rest > 0 else shown +``` + +- [ ] **Step 4: Run the tests** + +Run: `uv run pytest tests/unit/test_model_matching.py -q` +Expected: PASS + +- [ ] **Step 5: Commit** + +```bash +git add src/evalshift_cli/models/matching.py tests/unit/test_model_matching.py +git commit -m "feat(models): match recorded model ids against the source model + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 4: Promoted cases remember when and on what their capture ran + +**Files:** +- Modify: `src/evalshift_cli/captures/models.py` (`PromotedCase`) +- Modify: `src/evalshift_cli/captures/promote.py` (`capture_provenance`, `retire_promoted_case`, `__all__`) +- Modify: `src/evalshift_cli/cli/commands/capture.py` (`capture promote` + `capture sync` write the fields; `_load_promoted`, `_backfill_provenance`; summary clause) +- Modify: `tests/unit/fake_remote_store.py` (`capture_payload(model_id=...)`), `tests/unit/test_cli_capture.py` (`_events` / `_write_capture` gain `model_id`) +- Test: `tests/unit/test_captures_models.py`, `tests/unit/test_captures_promote.py`, `tests/unit/test_cli_capture.py` + +**Interfaces:** +- Produces: `PromotedCase.captured_at: str = ""`, `PromotedCase.source_model: str = ""`; `promote.capture_provenance(envelope: CaptureEnvelope) -> tuple[str, str]` (`(created_at, first model_call's model_id or "")`); `promote.retire_promoted_case(suite: str, name: str, *, base: Path | None = None) -> bool`; in `capture.py`: `_load_promoted(suite: str, *, base: Path | None) -> tuple[list[PromotedCase], str | None]` (cases, error type name), `_backfill_provenance(cases: list[PromotedCase], envelopes: Mapping[str, CaptureEnvelope], *, base: Path | None) -> tuple[list[PromotedCase], int]`; test helpers `capture_payload(..., model_id: str = "m")`, `_write_capture(..., model_id: str = "m")`. + +- [ ] **Step 1: Give the test helpers a model id** + +`tests/unit/fake_remote_store.py::capture_payload`: add keyword `model_id: str = "m"` and use it in place of the hardcoded `"model_id": "m"`. +`tests/unit/test_cli_capture.py`: add `model_id: str = "m"` to `_events` (used in its `"model_id"`) and to `_write_capture`, which passes it on: `_events(tools=..., model_input=model_input, model_id=model_id)`. + +- [ ] **Step 2: Write the failing tests** + +`tests/unit/test_captures_models.py` — append: + +```python +def test_a_case_written_before_capture_provenance_still_parses() -> None: + case = PromotedCase.model_validate( + {"name": "c", "suite": "s", "from_capture": "cap_1", "example": {"id": "c", "tools": []}} + ) + assert case.captured_at == "" + assert case.source_model == "" +``` + +(import `PromotedCase` from `evalshift_cli.captures.models` if the file does not already.) + +`tests/unit/test_captures_promote.py` — add `capture_provenance`, `retire_promoted_case`, `write_promoted_case` to the `evalshift_cli.captures.promote` import and `from evalshift_cli.suite.models import SuiteExample`, then append: + +```python +class TestCaptureProvenance: + def test_reads_created_at_and_the_first_model_calls_model(self) -> None: + envelope = _envelope( + events=[ + _model_call(0, model_input="hi", model_id="claude-sonnet-5-20260301"), + _model_call(1, model_input="again", model_id="claude-haiku-5"), + _final("done", 2), + ], + created_at="2026-10-01T12:00:00+00:00", + ) + assert capture_provenance(envelope) == ( + "2026-10-01T12:00:00+00:00", + "claude-sonnet-5-20260301", + ) + + def test_a_capture_without_a_model_call_has_no_model(self) -> None: + assert capture_provenance(_envelope(events=[_final("done", 0)]))[1] == "" + + +class TestRetirePromotedCase: + def test_deletes_the_case_file(self, tmp_path: Path) -> None: + case = PromotedCase( + name="cap_1", + suite="alpha", + from_capture="cap_1", + example=SuiteExample(id="cap_1", tools=[]), + ) + path = write_promoted_case(case, base=tmp_path) + assert retire_promoted_case("alpha", "cap_1", base=tmp_path) is True + assert not path.exists() + + def test_an_absent_case_is_not_an_error(self, tmp_path: Path) -> None: + assert retire_promoted_case("alpha", "cap_nope", base=tmp_path) is False +``` + +`tests/unit/test_cli_capture.py` — append: + +```python +def test_sync_records_when_and_on_what_each_capture_ran(tmp_path: Path) -> None: + _write_capture( + tmp_path, + capture_id="cap_1", + suite="alpha", + created_at="2026-10-01T09:30:00+00:00", + model_id="claude-sonnet-5-20260301", + ) + config = tmp_path / "evalshift.yaml" + _write_min_config(config) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + case = PromotedCase.model_validate_json( + (tmp_path / "suites" / "alpha" / "cap_1.json").read_text(encoding="utf-8") + ) + assert case.captured_at == "2026-10-01T09:30:00+00:00" + assert case.source_model == "claude-sonnet-5-20260301" + + +def test_sync_backfills_a_case_written_before_the_fields_existed(tmp_path: Path) -> None: + _write_capture(tmp_path, capture_id="cap_1", suite="alpha", model_id="claude-sonnet-5") + config = tmp_path / "evalshift.yaml" + _write_min_config(config) + assert _invoke(["sync", "--config", str(config)], tmp_path).exit_code == 0 + case_path = tmp_path / "suites" / "alpha" / "cap_1.json" + legacy = json.loads(case_path.read_text(encoding="utf-8")) + del legacy["captured_at"], legacy["source_model"] + case_path.write_text(json.dumps(legacy), encoding="utf-8") + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + case = PromotedCase.model_validate_json(case_path.read_text(encoding="utf-8")) + assert case.captured_at == "2026-06-16T12:00:00+00:00" + assert case.source_model == "claude-sonnet-5" + assert case.promoted_at == legacy["promoted_at"] + assert "backfilled capture time and model on 1 case(s)" in " ".join(result.stdout.split()) + + +def test_promote_records_when_and_on_what_the_capture_ran(tmp_path: Path) -> None: + _write_capture(tmp_path, capture_id="cap_1", model_id="gpt-5-2026-08-07") + + assert _invoke(["promote", "cap_1", "--as", "case1"], tmp_path).exit_code == 0 + + case = PromotedCase.model_validate_json( + (tmp_path / "suites" / "support_agent" / "case1.json").read_text(encoding="utf-8") + ) + assert case.source_model == "gpt-5-2026-08-07" + assert case.captured_at == "2026-06-16T12:00:00+00:00" +``` + +- [ ] **Step 3: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_captures_models.py tests/unit/test_captures_promote.py tests/unit/test_cli_capture.py -q -k "provenance or Provenance or Retire or records_when or backfills"` +Expected: FAIL — unknown attributes / missing imports. + +- [ ] **Step 4: Add the fields** + +In `src/evalshift_cli/captures/models.py`, inside `PromotedCase` after `cost_source`: + +```python + # When the source capture was recorded (its ``created_at``, verbatim) and the model its + # first ``model_call`` ran on -- the call a replay re-sends. With captures.max_age / + # captures.max_cases set, ``capture sync`` uses both to tell a stale case from a fresh + # one. Empty on case files written before they existed; sync backfills them while the + # capture is still local. + captured_at: str = "" + source_model: str = "" +``` + +- [ ] **Step 5: Add the promote helpers** + +In `src/evalshift_cli/captures/promote.py`, after `write_promoted_case`: + +```python +def capture_provenance(envelope: CaptureEnvelope) -> tuple[str, str]: + """``(captured_at, source_model)`` for a case promoted from ``envelope``. + + ``source_model`` is the first ``model_call``'s ``model_id`` -- the call whose input a replay + re-sends -- or ``""`` when the capture recorded none. + """ + first = next((e for e in envelope.trace.events if isinstance(e, ModelCallEvent)), None) + return envelope.created_at, first.model_id if first is not None else "" + + +def retire_promoted_case(suite: str, name: str, *, base: Path | None = None) -> bool: + """Delete ``/suites//.json``; ``False`` when it was already gone. + + The caller rebuilds ``golden.jsonl`` afterwards (:func:`rebuild_golden_jsonl`). + """ + path = promoted_suite_dir(suite, base=base) / f"{_safe_segment(name)}.json" + try: + path.unlink() + except FileNotFoundError: + return False + return True +``` + +Add both names to `__all__` (alphabetical order). + +- [ ] **Step 6: Write the fields from `promote` and `sync`, and backfill** + +In `src/evalshift_cli/cli/commands/capture.py` import `capture_provenance` from `evalshift_cli.captures.promote`. In **both** `PromotedCase(...)` constructions (`capture promote`, near line 725; `capture sync`, near line 1298) add, computed just before: + +```python + captured_at, source_model = capture_provenance(envelope) +``` + +and the keyword arguments `captured_at=captured_at, source_model=source_model,` (before `example=`). + +Add these helpers below `_promoted_content_owners`: + +```python +def _load_promoted(suite: str, *, base: Path | None) -> tuple[list[PromotedCase], str | None]: + """The suite's promoted cases, plus the error's type name when its files could not be read. + + A suite with no directory yet has no cases. Only the exception's type name is kept: its + text can carry raw bytes of a case file. + """ + suite_dir = promoted_suite_dir(suite, base=base) + if not suite_dir.is_dir(): + return [], None + try: + return iter_promoted_cases(suite_dir), None + except (ValueError, OSError) as exc: # pydantic's ValidationError is a ValueError + return [], type(exc).__name__ + + +def _backfill_provenance( + cases: list[PromotedCase], + envelopes: Mapping[str, CaptureEnvelope], + *, + base: Path | None, +) -> tuple[list[PromotedCase], int]: + """Fill ``captured_at`` / ``source_model`` on cases written before the fields existed. + + Only an empty field is filled, only from the case's own capture, and only while that + capture is local; nothing else on the case changes (``promoted_at`` included). Returns + the cases as they now stand and how many files were rewritten. + """ + out: list[PromotedCase] = [] + rewritten = 0 + for case in cases: + envelope = envelopes.get(case.from_capture) + if envelope is None or (case.captured_at and case.source_model): + out.append(case) + continue + captured_at, source_model = capture_provenance(envelope) + filled = case.model_copy( + update={ + "captured_at": case.captured_at or captured_at, + "source_model": case.source_model or source_model, + } + ) + if filled == case: + out.append(case) + continue + write_promoted_case(filled, base=base, force=True) + out.append(filled) + rewritten += 1 + return out, rewritten +``` + +In `capture_sync`: add `backfilled = 0` with the other counters. In the per-suite loop, right after the `duplicate_turn_warnings` loop and before `built_pairs = ...`, add: + +```python + envelopes = {r.envelope.capture_id: r.envelope for r in records_by_suite[suite_name]} + kept_cases, unreadable = _load_promoted(suite_name, base=base) + if unreadable is None: + kept_cases, rewritten = _backfill_provenance(kept_cases, envelopes, base=base) + backfilled += rewritten +``` + +In the summary block, right after the `wired_generation` clause: + +```python + if backfilled: + summary += f", backfilled capture time and model on {backfilled} case(s)" +``` + +- [ ] **Step 7: Run the new tests and the capture suites** + +Run: `uv run pytest tests/unit/test_captures_models.py tests/unit/test_captures_promote.py tests/unit/test_cli_capture.py tests/unit/test_cli_capture_remote.py tests/unit/test_cli_capture_repair.py tests/unit/test_cli_capture_native_shapes.py -q` +Expected: PASS. A test that compares a whole case file to a literal dict needs the two new keys (`""` or the fixture's values) — update it. + +- [ ] **Step 8: Commit** + +```bash +git add src/evalshift_cli/captures/models.py src/evalshift_cli/captures/promote.py src/evalshift_cli/cli/commands/capture.py tests/unit/fake_remote_store.py tests/unit/test_captures_models.py tests/unit/test_captures_promote.py tests/unit/test_cli_capture.py +git commit -m "feat(captures): promoted cases record their capture time and model + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 5: Staleness rules + +**Files:** +- Create: `src/evalshift_cli/captures/staleness.py` +- Test: `tests/unit/test_captures_staleness.py` (new) + +**Interfaces:** +- Consumes: `PromotedCase.captured_at/promoted_at/source_model` (Task 4), `same_model` (Task 3). +- Produces: `StaleReason = Literal["age", "model"]`; `NO_TOOLS_ROUTE = "∅"`; `parse_instant(text: str) -> datetime | None`; `case_instant(case: PromotedCase) -> datetime | None`; `is_reviewed(example: SuiteExample) -> bool`; `examples_route(examples: Iterable[SuiteExample]) -> str`; `@dataclass(frozen=True, slots=True) StalenessRules(now: datetime, max_age: timedelta | None = None, source_model: str | None = None)` with `stale_reason(instant: datetime | None, model: str) -> StaleReason | None`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/unit/test_captures_staleness.py`: + +```python +"""What makes a promoted case stale (spec §3).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.staleness import ( + NO_TOOLS_ROUTE, + StalenessRules, + case_instant, + examples_route, + is_reviewed, + parse_instant, +) +from evalshift_cli.suite.models import ExpectedToolCall, SuiteExample + +NOW = datetime(2026, 10, 16, 12, tzinfo=UTC) +#: A non-empty toolset (expected_tools may not sit beside an empty one). +_REF = "sha256:" + "cd" * 32 + + +def _example(*tools: str, reviewed: bool = False) -> SuiteExample: + calls = [ + ExpectedToolCall(tool_name=t, provenance="reviewed" if reviewed else "captured") + for t in tools + ] + return SuiteExample(id="c", toolset_ref=_REF, expected_tools=calls or None) + + +def _case(*, captured_at: str = "", promoted_at: str = "") -> PromotedCase: + return PromotedCase( + name="c", + suite="s", + from_capture="cap", + captured_at=captured_at, + promoted_at=promoted_at, + example=_example(), + ) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("2026-10-01T12:00:00+00:00", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ("2026-10-01T12:00:00Z", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ("2026-10-01T14:00:00+02:00", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ("2026-10-01T12:00:00", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ], +) +def test_parse_instant_normalises_to_utc(text: str, expected: datetime) -> None: + assert parse_instant(text) == expected + + +@pytest.mark.parametrize("text", ["", " ", "yesterday", "2026-13-01"]) +def test_parse_instant_is_none_for_anything_unparseable(text: str) -> None: + assert parse_instant(text) is None + + +def test_case_instant_prefers_the_capture_time() -> None: + case = _case(captured_at="2026-10-01T00:00:00+00:00", promoted_at="2026-10-05T00:00:00+00:00") + assert case_instant(case) == datetime(2026, 10, 1, tzinfo=UTC) + + +def test_case_instant_falls_back_to_the_promotion_time() -> None: + case = _case(promoted_at="2026-10-05T00:00:00+00:00") + assert case_instant(case) == datetime(2026, 10, 5, tzinfo=UTC) + + +def test_case_instant_is_none_when_neither_parses() -> None: + assert case_instant(_case()) is None + + +def test_is_reviewed_reads_any_expectation() -> None: + assert is_reviewed(_example("a", reviewed=True)) + assert not is_reviewed(_example("a")) + assert not is_reviewed(_example()) + + +def test_route_is_the_sorted_union_of_expected_tool_names() -> None: + assert examples_route([_example("b", "a")]) == "a,b" + assert examples_route([_example("a"), _example("c")]) == "a,c" + assert examples_route([_example()]) == NO_TOOLS_ROUTE + + +def test_an_old_case_is_age_stale() -> None: + rules = StalenessRules(now=NOW, max_age=timedelta(days=14)) + assert rules.stale_reason(NOW - timedelta(days=15), "m") == "age" + assert rules.stale_reason(NOW - timedelta(days=13), "m") is None + assert rules.stale_reason(None, "m") is None + + +def test_another_model_is_model_stale_before_age() -> None: + rules = StalenessRules(now=NOW, max_age=timedelta(days=14), source_model="claude-sonnet-5-5") + old = NOW - timedelta(days=30) + assert rules.stale_reason(old, "claude-sonnet-5-20260301") == "model" + assert rules.stale_reason(old, "claude-sonnet-5-5") == "age" + assert rules.stale_reason(NOW, "") is None + + +def test_no_limits_means_nothing_is_stale() -> None: + assert StalenessRules(now=NOW).stale_reason(NOW - timedelta(days=999), "x") is None +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_captures_staleness.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'evalshift_cli.captures.staleness'` + +- [ ] **Step 3: Implement `captures/staleness.py`** + +```python +"""What makes a promoted case stale -- shared by ``capture sync`` and the freshness checks. + +See ``docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md`` §3. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Literal + +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.models.matching import same_model +from evalshift_cli.suite.models import SuiteExample + +StaleReason = Literal["age", "model"] +"""Why a case is stale: older than ``captures.max_age``, or recorded on another model.""" + +#: The route of a case that expects no tool call. +NO_TOOLS_ROUTE = "∅" + + +def parse_instant(text: str) -> datetime | None: + """An ISO 8601 timestamp as an aware UTC datetime; naive means UTC; ``None`` if unparseable.""" + cleaned = text.strip() + if not cleaned: + return None + try: + parsed = datetime.fromisoformat(cleaned) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def case_instant(case: PromotedCase) -> datetime | None: + """When ``case``'s capture was recorded. + + ``captured_at``, else ``promoted_at`` -- an upper bound on the capture time, so a legacy + case looks younger than it is and is retired later, never earlier. + """ + return parse_instant(case.captured_at) or parse_instant(case.promoted_at) + + +def is_reviewed(example: SuiteExample) -> bool: + """Whether a human marked any of ``example``'s tool expectations ``provenance: reviewed``.""" + calls = list(example.expected_tools or []) + for round_calls in example.expected_tool_rounds or []: + calls.extend(round_calls) + return any(call.provenance == "reviewed" for call in calls) + + +def examples_route(examples: Iterable[SuiteExample]) -> str: + """The sorted, comma-joined round-1 tool names across ``examples``; ``∅`` for none.""" + names = sorted({call.tool_name for e in examples for call in e.expected_tools or []}) + return ",".join(names) if names else NO_TOOLS_ROUTE + + +@dataclass(frozen=True, slots=True) +class StalenessRules: + """The limits one sync (or one freshness check) measures cases against. + + Attributes: + now: The single instant ages are measured from. + max_age: ``captures.max_age``; ``None`` turns age staleness off. + source_model: The model a case must have been recorded on; ``None`` turns model + staleness off (no ``defaults.source_model``, or the guardrail tripped). + """ + + now: datetime + max_age: timedelta | None = None + source_model: str | None = None + + def stale_reason(self, instant: datetime | None, model: str) -> StaleReason | None: + """``model`` before ``age``; ``None`` when fresh or when the fact is unknown.""" + if self.source_model and model and not same_model(model, self.source_model): + return "model" + if self.max_age is not None and instant is not None and instant < self.now - self.max_age: + return "age" + return None +``` + +- [ ] **Step 4: Run the tests** + +Run: `uv run pytest tests/unit/test_captures_staleness.py -q && uv run mypy --strict src/evalshift_cli` +Expected: PASS / clean. (If importing `captures.staleness` from `captures/__init__` order causes a cycle, do not add it to `captures/__init__.py` — import it by module path.) + +- [ ] **Step 5: Commit** + +```bash +git add src/evalshift_cli/captures/staleness.py tests/unit/test_captures_staleness.py +git commit -m "feat(captures): staleness rules for promoted cases + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 6: The per-suite planner + +**Files:** +- Create: `src/evalshift_cli/captures/selection.py` +- Test: `tests/unit/test_captures_selection.py` (new) + +**Interfaces:** +- Consumes: `StaleReason` (Task 5). +- Produces: `RetireReason = Literal["age", "model", "over_cap"]`; frozen dataclasses `KeptCase(name: str, instant: datetime | None, stale: StaleReason | None, reviewed: bool, route: str, content_key: str)`, `CandidateGroup(key: str, capture_ids: tuple[str, ...], route: str, content_keys: frozenset[str])` with property `size: int`, `SuitePlan(add: tuple[CandidateGroup, ...] = (), retire: tuple[tuple[str, RetireReason], ...] = (), reviewed_over_cap: int = 0)` with property `added_cases: int`; `plan_suite(kept: Sequence[KeptCase], candidates: Sequence[CandidateGroup], *, max_cases: int) -> SuitePlan`. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/unit/test_captures_selection.py`: + +```python +"""``plan_suite``: which captures enter a bounded suite and which cases leave (spec §5).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from evalshift_cli.captures.selection import CandidateGroup, KeptCase, SuitePlan, plan_suite +from evalshift_cli.captures.staleness import StaleReason + +T0 = datetime(2026, 10, 1, tzinfo=UTC) + + +def _kept( + name: str, + *, + days: int = 0, + stale: StaleReason | None = None, + reviewed: bool = False, + route: str = "a", + key: str | None = None, +) -> KeptCase: + return KeptCase( + name=name, + instant=T0 + timedelta(days=days), + stale=stale, + reviewed=reviewed, + route=route, + content_key=key or f"k-{name}", + ) + + +def _group( + key: str, *, route: str = "a", size: int = 1, content: set[str] | None = None +) -> CandidateGroup: + ids = tuple(f"{key}-t{i}" for i in range(size)) if size > 1 else (key,) + return CandidateGroup( + key=key, + capture_ids=ids, + route=route, + content_keys=frozenset(content or {f"k-{i}" for i in ids}), + ) + + +def _added(plan: SuitePlan) -> list[str]: + return [g.key for g in plan.add] + + +def test_fills_free_slots_up_to_the_cap() -> None: + plan = plan_suite([], [_group(f"c{i}") for i in range(5)], max_cases=3) + assert plan.added_cases == 3 + assert plan.retire == () + + +def test_the_plan_is_deterministic_whatever_the_input_order() -> None: + groups = [_group(f"c{i}", route="ab"[i % 2]) for i in range(7)] + assert plan_suite([], groups, max_cases=4) == plan_suite([], groups[::-1], max_cases=4) + + +def test_the_least_represented_route_fills_first() -> None: + kept = [_kept(f"f{i}") for i in range(3)] + groups = [_group(f"a{i}") for i in range(5)] + [_group(f"b{i}", route="b") for i in range(2)] + plan = plan_suite(kept, groups, max_cases=6) + assert [g.route for g in plan.add] == ["b", "b", "a"] + + +def test_a_group_is_added_whole_or_not_at_all() -> None: + plan = plan_suite([], [_group("conv", size=3), _group("solo", route="b")], max_cases=2) + assert _added(plan) == ["solo"] + assert plan.added_cases == 1 + + +def test_stale_cases_stay_when_nothing_replaces_them() -> None: + kept = [_kept(f"s{i}", stale="age") for i in range(3)] + assert plan_suite(kept, [], max_cases=3) == SuitePlan() + + +def test_stale_cases_leave_only_to_make_room_oldest_first() -> None: + kept = [_kept("s1", days=0, stale="age"), _kept("s2", days=1, stale="age"), _kept("s3", days=2, stale="age")] + plan = plan_suite(kept, [_group("new")], max_cases=3) + assert _added(plan) == ["new"] + assert plan.retire == (("s1", "age"),) + + +def test_another_model_leaves_before_old_age() -> None: + kept = [_kept("old", days=0, stale="age"), _kept("mod", days=5, stale="model")] + plan = plan_suite(kept, [_group("new")], max_cases=2) + assert plan.retire == (("mod", "model"),) + + +def test_a_stale_twin_is_replaced_even_under_the_cap() -> None: + kept = [_kept("twin", stale="age", key="k-q")] + plan = plan_suite(kept, [_group("new", content={"k-q"})], max_cases=10) + assert _added(plan) == ["new"] + assert plan.retire == (("twin", "age"),) + + +def test_reviewed_cases_are_never_retired() -> None: + kept = [_kept("r1", stale="age", reviewed=True), _kept("r2", stale="model", reviewed=True)] + plan = plan_suite(kept, [_group(f"c{i}") for i in range(3)], max_cases=1) + assert plan.add == () + assert plan.retire == () + assert plan.reviewed_over_cap == 1 + + +def test_an_over_cap_suite_is_trimmed_from_its_largest_route_oldest_first() -> None: + kept = [ + _kept("a1", days=0), + _kept("a2", days=1), + _kept("a3", days=2), + _kept("b1", days=0, route="b"), + ] + plan = plan_suite(kept, [], max_cases=2) + assert plan.retire == (("a1", "over_cap"), ("a2", "over_cap")) + + +def test_planning_twice_changes_nothing() -> None: + kept = [_kept(f"f{i}") for i in range(3)] + assert plan_suite(kept, [_group("c1"), _group("c2")], max_cases=3) == SuitePlan() + + +@pytest.mark.parametrize("max_cases", [1, 2, 3, 5, 8]) +def test_a_suite_never_ends_over_the_cap_unless_reviewed_cases_force_it(max_cases: int) -> None: + kept = [ + _kept("r", reviewed=True), + _kept("f1"), + _kept("f2", route="b"), + _kept("s1", stale="age"), + _kept("s2", stale="model", route="b"), + ] + groups = [_group("c1"), _group("c2", route="c"), _group("conv", size=2, route="b")] + plan = plan_suite(kept, groups, max_cases=max_cases) + names = [name for name, _ in plan.retire] + assert len(names) == len(set(names)) + assert "r" not in names + assert len(kept) - len(plan.retire) + plan.added_cases <= max_cases +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_captures_selection.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'evalshift_cli.captures.selection'` + +- [ ] **Step 3: Implement `captures/selection.py`** + +```python +"""Which captures enter a bounded suite and which promoted cases leave it. + +Pure: no I/O, no clock. ``capture sync`` builds the inputs from disk and applies the plan. +See ``docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md`` §5. +""" + +from __future__ import annotations + +import hashlib +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from evalshift_cli.captures.staleness import StaleReason + +RetireReason = Literal["age", "model", "over_cap"] +"""Why a case left: too old, recorded on another model, or trimmed to ``max_cases``.""" + + +@dataclass(frozen=True, slots=True) +class KeptCase: + """A promoted case already in the suite, as the planner sees it. + + Attributes: + name: The case name (its file stem) -- what a retirement deletes. + instant: When its capture was recorded (``staleness.case_instant``), if known. + stale: Why it is stale, or ``None`` when fresh. + reviewed: A human marked one of its expectations ``reviewed``: never retired. + route: Its tool route (``staleness.examples_route``). + content_key: ``promote.example_content_key`` of its example. + """ + + name: str + instant: datetime | None + stale: StaleReason | None + reviewed: bool + route: str + content_key: str + + +@dataclass(frozen=True, slots=True) +class CandidateGroup: + """Eligible captures that enter a suite together: one conversation, or one lone capture. + + Attributes: + key: Stable identity used for the deterministic order. + capture_ids: Its turns' case names (the capture ids), in turn order. + route: The union route of its turns. + content_keys: Each turn's ``example_content_key``. + """ + + key: str + capture_ids: tuple[str, ...] + route: str + content_keys: frozenset[str] + + @property + def size(self) -> int: + """How many cases the group adds.""" + return len(self.capture_ids) + + +@dataclass(frozen=True, slots=True) +class SuitePlan: + """What one sync does to one suite. + + Attributes: + add: Groups to promote, in the order they were picked. + retire: ``(case name, reason)`` for every case to delete. + reviewed_over_cap: Reviewed cases beyond ``max_cases``, kept anyway. + """ + + add: tuple[CandidateGroup, ...] = () + retire: tuple[tuple[str, RetireReason], ...] = () + reviewed_over_cap: int = 0 + + @property + def added_cases(self) -> int: + """How many cases ``add`` promotes.""" + return sum(group.size for group in self.add) + + +def _hash_order(key: str) -> str: + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + +def _when(case: KeptCase) -> float: + """Sort key, oldest first; an unknown instant counts as oldest of all.""" + return case.instant.timestamp() if case.instant is not None else float("-inf") + + +def plan_suite( + kept: Sequence[KeptCase], + candidates: Sequence[CandidateGroup], + *, + max_cases: int, +) -> SuitePlan: + """Plan one suite: fill free slots route-balanced; retire stale cases only to make room. + + 1. Reviewed cases are pinned; every other case is fresh or stale. + 2. Free slots (``max_cases`` minus pinned and fresh) are filled one group at a time from + the route least represented so far (ties by route name), each route's groups in + ``sha256(key)`` order. A group is added whole, or skipped when it does not fit. + 3. A stale case whose content an added turn replays is retired outright, so no duplicate + content survives. Other stale cases leave only while the suite is over ``max_cases``: + recorded-on-another-model first, then oldest. + 4. Still over (a legacy or over-cap suite): fresh cases are trimmed from the most + represented route, oldest first. Reviewed cases never leave. + + Same inputs, same plan, on any machine. + """ + pinned = [c for c in kept if c.reviewed] + fresh = [c for c in kept if not c.reviewed and c.stale is None] + stale = [c for c in kept if not c.reviewed and c.stale is not None] + + counts: Counter[str] = Counter(c.route for c in (*pinned, *fresh)) + queues: dict[str, list[CandidateGroup]] = {} + for group in sorted(candidates, key=lambda g: _hash_order(g.key)): + queues.setdefault(group.route, []).append(group) + + slots = max(0, max_cases - len(pinned) - len(fresh)) + added: list[CandidateGroup] = [] + while slots > 0: + fitting = [r for r, queue in queues.items() if any(g.size <= slots for g in queue)] + if not fitting: + break + route = min(fitting, key=lambda r: (counts[r], r)) + queue = queues[route] + group = queue.pop(next(i for i, g in enumerate(queue) if g.size <= slots)) + added.append(group) + counts[route] += group.size + slots -= group.size + + retire: list[tuple[str, RetireReason]] = [] + replayed = frozenset(key for group in added for key in group.content_keys) + remaining: list[KeptCase] = [] + for case in stale: + if case.stale is not None and case.content_key in replayed: + retire.append((case.name, case.stale)) + else: + remaining.append(case) + + excess = len(pinned) + len(fresh) + sum(g.size for g in added) + len(remaining) - max_cases + for case in sorted(remaining, key=lambda c: (c.stale != "model", _when(c), c.name)): + if excess <= 0: + break + if case.stale is not None: + retire.append((case.name, case.stale)) + excess -= 1 + + pool = sorted(fresh, key=lambda c: (_when(c), c.name)) + while excess > 0 and pool: + route = min({c.route for c in pool}, key=lambda r: (-counts[r], r)) + victim = next(c for c in pool if c.route == route) + pool.remove(victim) + counts[route] -= 1 + retire.append((victim.name, "over_cap")) + excess -= 1 + + return SuitePlan( + add=tuple(added), + retire=tuple(retire), + reviewed_over_cap=max(0, len(pinned) - max_cases), + ) +``` + +- [ ] **Step 4: Run the tests** + +Run: `uv run pytest tests/unit/test_captures_selection.py -q && uv run mypy --strict src/evalshift_cli` +Expected: PASS / clean. + +- [ ] **Step 5: Commit** + +```bash +git add src/evalshift_cli/captures/selection.py tests/unit/test_captures_selection.py +git commit -m "feat(captures): plan_suite picks and retires cases for a bounded suite + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 7: Sync reads the limits; the fetch window follows `max_age` + +**Files:** +- Modify: `src/evalshift_cli/captures/remote.py` (`FetchSummary.since_label`, `fetch_captures(since_label=...)`) +- Modify: `src/evalshift_cli/cli/commands/capture.py` (`_now`, `_SyncLimits`, `_sync_limits`, `_maybe_fetch(window=...)`, `_fetch_into_base(since_label=...)`, top of `capture_sync`) +- Test: `tests/unit/test_captures_remote.py`, `tests/unit/test_cli_capture_remote.py` + +**Interfaces:** +- Consumes: `CapturesConfig.max_age/max_cases` (Task 1), `format_duration` (Task 1). +- Produces: `FetchSummary.since_label: str = "--since"`; `fetch_captures(..., since_label: str = "--since")`; in `capture.py`: `_now() -> datetime`, `@dataclass(frozen=True, slots=True) _SyncLimits(max_age: timedelta | None = None, max_cases: int | None = None, source_model: str | None = None)` with property `active: bool`; `_sync_limits(config_path: Path, *, console: Console, warn: bool) -> _SyncLimits`; inside `capture_sync` the locals `limits: _SyncLimits` and `now: datetime` that Task 8 uses. + +- [ ] **Step 1: Write the failing tests** + +`tests/unit/test_captures_remote.py` — append (import `FetchSummary` if needed): + +```python +def test_skipped_old_names_the_window_it_was_measured_against() -> None: + summary = FetchSummary( + store_uri="s3://b/p", skipped_old=3, since_label="captures.max_age (14d)" + ) + assert "3 older than captures.max_age (14d)" in summary.describe() + + +def test_skipped_old_defaults_to_the_since_flag() -> None: + assert "1 older than --since" in FetchSummary(store_uri="s3://b/p", skipped_old=1).describe() +``` + +`tests/unit/test_cli_capture_remote.py` — append (import `datetime`, `UTC`): + +```python +_NOW = datetime(2026, 6, 20, tzinfo=UTC) + + +def _with_max_age(config: Path) -> Path: + config.write_text(config.read_text(encoding="utf-8") + " max_age: 14d\n", encoding="utf-8") + return config + + +def test_sync_fetches_only_inside_max_age( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, remote: FakeRemoteStore +) -> None: + remote.modified["captures/alpha/cap_1.json"] = datetime(2026, 5, 1, tzinfo=UTC) + config = _with_max_age(_write_config(tmp_path / "evalshift.yaml")) + monkeypatch.setattr(capture_module, "_now", lambda: _NOW) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + flat = " ".join(result.stdout.split()) + assert "fetched 1 capture(s)" in flat + assert "1 older than captures.max_age (14d)" in flat + assert not (tmp_path / "captures" / "alpha" / "cap_1.json").exists() + + +def test_an_explicit_since_overrides_max_age( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, remote: FakeRemoteStore +) -> None: + remote.modified["captures/alpha/cap_1.json"] = datetime(2026, 5, 1, tzinfo=UTC) + config = _with_max_age(_write_config(tmp_path / "evalshift.yaml")) + monkeypatch.setattr(capture_module, "_now", lambda: _NOW) + + result = _invoke(["sync", "--since", "2026-04-01", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert "fetched 2 capture(s)" in result.stdout + + +def test_max_age_without_a_store_is_not_a_fetch_error(tmp_path: Path) -> None: + config = _write_config(tmp_path / "evalshift.yaml", store=None) + config.write_text( + config.read_text(encoding="utf-8") + "\ncaptures:\n max_age: 14d\n", encoding="utf-8" + ) + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + + +def test_offline_sync_says_it_ignores_limits_it_cannot_read(tmp_path: Path) -> None: + config = tmp_path / "evalshift.yaml" + config.write_text("captures:\n max_age: 2w\n", encoding="utf-8") + result = _invoke(["sync", "--offline", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "syncing without captures.max_age / captures.max_cases" in " ".join( + result.stdout.split() + ) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_captures_remote.py tests/unit/test_cli_capture_remote.py -q -k "window or since_flag or max_age or offline_sync_says"` +Expected: FAIL — `FetchSummary` has no `since_label`; `capture_module` has no `_now`. + +- [ ] **Step 3: Label the skip** + +In `captures/remote.py`: add `since_label: str = "--since"` as the last field of `FetchSummary`; in `describe()` change the `(self.skipped_old, "older than --since")` pair to `(self.skipped_old, f"older than {self.since_label}")`; add keyword `since_label: str = "--since"` to `fetch_captures` (document it in the docstring: "how the summary names the ``since`` cutoff") and construct `FetchSummary(store_uri=store.uri, since_label=since_label)`. + +- [ ] **Step 4: Read the limits in sync and pass the window** + +In `cli/commands/capture.py` add `from dataclasses import dataclass`, `timedelta` to the `datetime` import, and `from evalshift_cli.utils.duration import format_duration`. Replace `_now_iso` with: + +```python +def _now() -> datetime: + """The one instant a sync measures capture age from (tests pin it).""" + return datetime.now(UTC) + + +def _now_iso() -> str: + return _now().isoformat() +``` + +Below `_warn_store_skipped` add: + +```python +@dataclass(frozen=True, slots=True) +class _SyncLimits: + """``captures.max_age`` / ``captures.max_cases`` and ``defaults.source_model``, for sync. + + ``active`` is the opt-in switch (spec §1): with neither limit set, sync behaves exactly as + it did before the limits existed, whatever ``defaults.source_model`` says. + """ + + max_age: timedelta | None = None + max_cases: int | None = None + source_model: str | None = None + + @property + def active(self) -> bool: + return self.max_age is not None or self.max_cases is not None + + +def _sync_limits(config_path: Path, *, console: Console, warn: bool) -> _SyncLimits: + """Read the limits from ``config_path``; none at all when it is missing or unreadable. + + ``warn`` is for ``--offline``: online, the fetch step has already loaded the same file and + warned about it, so a second line would only repeat it. + """ + if not config_path.exists(): + return _SyncLimits() + try: + cfg = load_config(config_path) + except ConfigError as exc: + reason = exc.summary + except (OSError, UnicodeDecodeError) as exc: + reason = type(exc).__name__ # its text can carry raw bytes of the file + else: + return _SyncLimits( + max_age=cfg.captures.max_age, + max_cases=cfg.captures.max_cases, + source_model=cfg.defaults.source_model, + ) + if warn and _mentions_captures(config_path): + console.print( + f"[yellow]⚠[/yellow] could not read {escape(str(config_path))} ({escape(reason)}); " + "syncing without captures.max_age / captures.max_cases.", + ) + return _SyncLimits() +``` + +`_fetch_into_base`: add keyword `since_label: str = "--since"` and pass `since_label=since_label` to `fetch_captures`. + +`_maybe_fetch`: add keyword `window: tuple[datetime, str] | None = None` (docstring: "the cutoff ``captures.max_age`` implies, and how to name it; an explicit ``--since`` wins") and replace its body with: + +```python + since_dt = _resolve_since(since) + since_label = "--since" + if since_dt is None and window is not None: + since_dt, since_label = window + store = None if offline else _configured_store(config_path, console=console, strict=False) + if store is None: + if since is not None: + console.print( + "[red]✗[/red] --since needs captures.store in evalshift.yaml and no --offline: " + "there is nothing to fetch from.", + ) + raise typer.Exit(code=1) + return + _fetch_into_base( + store, + base=base, + suite=suite, + since=since_dt, + since_label=since_label, + console=console, + force_ids=force_ids, + ) +``` + +At the top of `capture_sync`, between `_check_rounds(rounds)` and `selection = ...`: + +```python + limits = _sync_limits(config_path, console=console, warn=offline) + now = _now() + window = ( + (now - limits.max_age, f"captures.max_age ({format_duration(limits.max_age)})") + if limits.max_age is not None + else None + ) +``` + +and pass `window=window` in the `_maybe_fetch(...)` call. + +- [ ] **Step 5: Run the tests** + +Run: `uv run pytest tests/unit/test_captures_remote.py tests/unit/test_cli_capture_remote.py tests/unit/test_cli_capture.py -q && uv run mypy --strict src/evalshift_cli` +Expected: PASS / clean. + +- [ ] **Step 6: Commit** + +```bash +git add src/evalshift_cli/captures/remote.py src/evalshift_cli/cli/commands/capture.py tests/unit/test_captures_remote.py tests/unit/test_cli_capture_remote.py +git commit -m "feat(capture): sync reads captures limits; max_age bounds the fetch + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 8: Sync filters, plans and retires + +**Files:** +- Modify: `src/evalshift_cli/cli/commands/capture.py` (`capture_sync` + helpers `_suites_with_cases`, `_suite_rules`, `_candidate_groups`, `_reason_counts`, `_plan_line`) +- Test: `tests/unit/test_cli_capture_limits.py` (new) + +**Interfaces:** +- Consumes: `_SyncLimits`, `limits`, `now` (Task 7); `_load_promoted`, `_backfill_provenance`, `capture_provenance`, `retire_promoted_case` (Task 4); `StalenessRules`, `StaleReason`, `case_instant`, `examples_route`, `is_reviewed`, `parse_instant` (Task 5); `plan_suite`, `KeptCase`, `CandidateGroup`, `SuitePlan` (Task 6); `model_filter_applies`, `describe_models` (Task 3). +- Produces: the sync behaviour and output lines in Global Constraints. No new public API. + +- [ ] **Step 1: Write the failing tests** + +Create `tests/unit/test_cli_capture_limits.py`: + +```python +"""``capture sync`` with ``captures.max_age`` / ``captures.max_cases`` set (bounded suites).""" + +from __future__ import annotations + +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from evalshift_cli.cli.commands import capture as capture_module +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config +from evalshift_cli.cli.main import app +from evalshift_cli.suite.loader import load_jsonl +from tests.unit.fake_remote_store import TOOLSET_HEX, TOOLSET_PAYLOAD, capture_payload + +runner = CliRunner() + +WEEK_1 = "2026-06-10T12:00:00+00:00" +WEEK_3 = "2026-06-29T12:00:00+00:00" + + +def _at(monkeypatch: pytest.MonkeyPatch, day: int) -> None: + """Pin sync's clock to noon UTC on 2026-06-.""" + monkeypatch.setattr(capture_module, "_now", lambda: datetime(2026, 6, day, 12, tzinfo=UTC)) + + +def _put( + base: Path, + cid: str, + *, + suite: str = "alpha", + created_at: str = WEEK_1, + model_id: str = "claude-sonnet-5-20260301", + text: str | None = None, + conversation_id: str | None = None, + turn_index: int | None = None, +) -> None: + """Write one local capture and the shared toolset sidecar it points at.""" + sidecar = base / "toolsets" / f"{TOOLSET_HEX}.json" + sidecar.parent.mkdir(parents=True, exist_ok=True) + sidecar.write_bytes(TOOLSET_PAYLOAD) + path = base / "captures" / suite / f"{cid}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes( + capture_payload( + cid, + suite, + model_input=text if text is not None else f"question {cid}?", + created_at=created_at, + model_id=model_id, + conversation_id=conversation_id, + turn_index=turn_index, + ) + ) + + +def _config( + path: Path, + *, + max_age: str | None = "14d", + max_cases: int | None = 3, + source_model: str = "claude-sonnet-5", +) -> Path: + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + text = re.sub(r"(?m)^ source_model: .*$", f" source_model: {source_model}", text) + limits = (f" max_age: {max_age}\n" if max_age else "") + ( + f" max_cases: {max_cases}\n" if max_cases else "" + ) + if limits: + text += "\ncaptures:\n" + limits + path.write_text(text, encoding="utf-8") + return path + + +def _sync(tmp_path: Path, *args: str) -> Any: + result = runner.invoke( + app, + [ + "capture", + "sync", + "--config", + str(tmp_path / "evalshift.yaml"), + "--base", + str(tmp_path), + *args, + ], + ) + assert result.exit_code == 0, result.stdout + return result + + +def _flat(result: Any) -> str: + return " ".join(result.stdout.split()) + + +def _cases(tmp_path: Path, suite: str = "alpha") -> set[str]: + return {p.stem for p in (tmp_path / "suites" / suite).glob("*.json")} + + +def test_a_first_sync_caps_the_suite_at_max_cases( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert len(load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").examples) == 3 + assert "alpha: 3 cases — kept 0, retired 0, added 3 of 5 eligible" in _flat(result) + + +def test_syncing_again_with_nothing_new_changes_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + suite_dir = tmp_path / "suites" / "alpha" + before = {p.name: p.read_bytes() for p in suite_dir.iterdir()} + + result = _sync(tmp_path) + + assert {p.name: p.read_bytes() for p in suite_dir.iterdir()} == before + assert "alpha: 3 cases — kept 3, retired 0, added 0 of 2 eligible" in _flat(result) + + +def test_stale_cases_are_swapped_for_fresh_captures( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"old_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new_0", created_at=WEEK_3) + _put(tmp_path, "new_1", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"old_2", "new_0", "new_1"} + assert ( + "alpha: 3 cases — kept 1, retired 2 (2 older than 14d), added 2 of 2 eligible" + in _flat(result) + ) + + +def test_a_stale_suite_stays_full_when_nothing_replaces_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"old_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert "alpha: 3 cases — kept 3, retired 0, added 0 of 0 eligible" in _flat(result) + + +def test_cases_recorded_on_the_old_model_leave_after_a_migration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"old_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new_0", created_at="2026-06-13T12:00:00+00:00", model_id="claude-sonnet-5-5") + _config(tmp_path / "evalshift.yaml", source_model="claude-sonnet-5-5") + _at(monkeypatch, 14) + + result = _sync(tmp_path) + + assert "new_0" in _cases(tmp_path) + assert len(_cases(tmp_path)) == 3 + assert "retired 1 (1 on another model), added 1 of 1 eligible" in _flat(result) + + +def test_a_source_model_no_capture_ran_on_filters_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", source_model="gpt-9") + _at(monkeypatch, 12) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + flat = _flat(result) + assert "alpha: none of 3 case(s) and capture(s) were recorded on gpt-9" in flat + assert "(seen: claude-sonnet-5-20260301 ×3)" in flat + assert "not filtering by model — check defaults.source_model" in flat + + +def test_captures_outside_the_window_are_not_eligible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _put(tmp_path, "old") + _put(tmp_path, "new", created_at=WEEK_3) + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"new"} + assert "added 1 of 1 eligible; not eligible: 1 older than 14d" in _flat(result) + + +def test_max_age_alone_filters_without_capping( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"new_{i}", created_at=WEEK_3) + _put(tmp_path, "old") + _config(tmp_path / "evalshift.yaml", max_cases=None) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 5 + assert "alpha: not eligible: 1 older than 14d" in _flat(result) + + +def test_an_over_cap_suite_is_trimmed_without_new_captures( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + for path in (tmp_path / "captures" / "alpha").glob("*.json"): + path.unlink() + _config(tmp_path / "evalshift.yaml", max_cases=3) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert len(load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").examples) == 3 + assert ( + "alpha: 3 cases — kept 3, retired 2 (2 over max_cases), added 0 of 0 eligible" + in _flat(result) + ) + + +def test_suite_flag_leaves_other_suites_alone( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for suite in ("alpha", "beta"): + for i in range(5): + _put(tmp_path, f"{suite}_{i}", suite=suite) + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + _config(tmp_path / "evalshift.yaml", max_cases=3) + + _sync(tmp_path, "--suite", "alpha") + + assert len(_cases(tmp_path, "alpha")) == 3 + assert len(_cases(tmp_path, "beta")) == 5 + + +def test_a_conversation_enters_whole_or_not_at_all( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for turn in range(3): + _put(tmp_path, f"conv_t{turn}", conversation_id="c1", turn_index=turn) + _put(tmp_path, "solo") + _config(tmp_path / "evalshift.yaml", max_cases=2) + _at(monkeypatch, 12) + + _sync(tmp_path) + + assert _cases(tmp_path) == {"solo"} + + +def test_the_same_question_captured_fresh_replaces_its_stale_case( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _put(tmp_path, "old", text="where is order 1?") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"new"} + assert "retired 1 (1 older than 14d), added 1 of 1 eligible" in _flat(result) + + +def test_without_limits_sync_promotes_everything_as_before( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None, source_model="gpt-9") + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 5 + assert "cases — kept" not in _flat(result) + assert "not filtering by model" not in _flat(result) +``` + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_cli_capture_limits.py -q` +Expected: FAIL — suites are not capped, no per-suite lines (`test_without_limits_sync_promotes_everything_as_before` already passes). + +- [ ] **Step 3: Add the helpers** + +In `cli/commands/capture.py` add imports: + +```python +from collections import Counter +from collections.abc import Sequence # extend the existing collections.abc import +from evalshift_cli.captures.promote import retire_promoted_case # extend the existing import +from evalshift_cli.captures.selection import CandidateGroup, KeptCase, SuitePlan, plan_suite +from evalshift_cli.captures.staleness import ( + StaleReason, + StalenessRules, + case_instant, + examples_route, + is_reviewed, + parse_instant, +) +from evalshift_cli.models.matching import describe_models, model_filter_applies +``` + +and below `_backfill_provenance`: + +```python +def _suites_with_cases(*, base: Path | None, only: str | None) -> set[str]: + """Suites under ``/suites/`` holding at least one case file (``only`` narrows).""" + root = suites_root(base) + if not root.is_dir(): + return set() + return { + path.name + for path in root.iterdir() + if path.is_dir() and any(path.glob("*.json")) and (only is None or path.name == only) + } + + +def _suite_rules( + suite: str, + limits: _SyncLimits, + *, + now: datetime, + recorded_models: list[str], + console: Console, +) -> StalenessRules: + """One suite's staleness rules, with the model guardrail applied (spec §3).""" + source = limits.source_model + if source and not model_filter_applies(recorded_models, source): + known = [m for m in recorded_models if m] + if known: + console.print( + f"[yellow]⚠[/yellow] {escape(suite)}: none of {len(known)} case(s) and " + f"capture(s) were recorded on {escape(source)} " + f"(seen: {escape(describe_models(known))}); not filtering by model — " + "check defaults.source_model", + soft_wrap=True, + ) + source = None + return StalenessRules(now=now, max_age=limits.max_age, source_model=source) + + +def _candidate_groups(pending: Sequence[PromotedCase]) -> list[CandidateGroup]: + """Group pending cases by conversation (a lone case is its own group), turn order kept.""" + turns: dict[str, list[PromotedCase]] = {} + for case in pending: + key = ( + f"conversation:{case.conversation_id}" + if case.conversation_id + else f"capture:{case.name}" + ) + turns.setdefault(key, []).append(case) + return [ + CandidateGroup( + key=key, + capture_ids=tuple(c.name for c in cases), + route=examples_route(c.example for c in cases), + content_keys=frozenset(example_content_key(c.example) for c in cases), + ) + for key, cases in turns.items() + ] + + +def _reason_counts(counts: Counter[str], *, max_age: timedelta | None) -> str: + """``2 older than 14d, 1 on another model, 3 over max_cases``; zero counts left out.""" + labels = { + "age": f"older than {format_duration(max_age)}" if max_age is not None else "too old", + "model": "on another model", + "over_cap": "over max_cases", + } + return ", ".join(f"{counts[k]} {labels[k]}" for k in ("age", "model", "over_cap") if counts[k]) + + +def _plan_line( + suite: str, + plan: SuitePlan | None, + *, + kept: int, + eligible: int, + ineligible: Counter[str], + max_age: timedelta | None, +) -> str | None: + """The per-suite summary line (spec §6); ``plan`` is ``None`` with ``max_age`` alone.""" + not_eligible = _reason_counts(ineligible, max_age=max_age) + if plan is None: + return f" {suite}: not eligible: {not_eligible}" if not_eligible else None + retired = Counter(reason for _, reason in plan.retire) + detail = _reason_counts(retired, max_age=max_age) + stayed = kept - len(plan.retire) + return ( + f" {suite}: {stayed + plan.added_cases} cases — kept {stayed}, " + f"retired {len(plan.retire)}" + + (f" ({detail})" if detail else "") + + f", added {plan.added_cases} of {eligible} eligible" + + (f"; not eligible: {not_eligible}" if not_eligible else "") + ) +``` + +- [ ] **Step 4: Wire them into `capture_sync`** + +Edit `capture_sync` in this order (anchors are existing code): + +(a) The early return becomes: + +```python + if not records and not ( + limits.max_cases is not None and _suites_with_cases(base=base, only=suite) + ): + where = f" for suite {suite!r}" if suite else "" + console.print(f"[dim]no captures found{where} under .evalshift/captures/.[/dim]") + return +``` + +(b) Next to the other counters add `plan_lines: list[str] = []`. + +(c) Replace `for suite_name in sorted(records_by_suite):` with: + +```python + planned = set(records_by_suite) + if limits.max_cases is not None: + # An over-cap suite is trimmed even when no new capture arrived for it. + planned |= _suites_with_cases(base=base, only=suite) + suites_seen |= planned + for suite_name in sorted(planned): + suite_records = records_by_suite.get(suite_name, []) +``` + +and inside the loop replace every `records_by_suite[suite_name]` with `suite_records`. + +(d) Right after the Task 4 backfill lines (`if unreadable is None: ...`), add: + +```python + if unreadable is not None and limits.active: + console.print( + f"[yellow]⚠[/yellow] {escape(suite_name)}: could not read its promoted cases " + f"({unreadable}); left it unchanged.", + ) + continue + kept_from = {c.from_capture for c in kept_cases} + stale_by_name: dict[str, StaleReason | None] = {} + rules: StalenessRules | None = None + if limits.active: + recorded = [c.source_model for c in kept_cases] + [ + capture_provenance(r.envelope)[1] + for r in suite_records + if r.envelope.capture_id not in kept_from + ] + rules = _suite_rules( + suite_name, limits, now=now, recorded_models=recorded, console=console + ) + stale_by_name = { + c.name: rules.stale_reason(case_instant(c), c.source_model) for c in kept_cases + } + ineligible: Counter[str] = Counter() + pending: list[PromotedCase] = [] +``` + +(e) Replace the disk seed `seen_examples.update(_promoted_content_owners(suite_name, base=base, console=console))` with: + +```python + owners = _promoted_content_owners(suite_name, base=base, console=console) + # A stale case does not claim its content: the same question captured fresh + # replaces it (spec §5, rule 3). + seen_examples.update( + (key, owner) for key, owner in owners.items() if stale_by_name.get(owner) is None + ) +``` + +(f) In the per-record loop, after the `if built.blocked is not None: ... continue` block and before `content_key = (suite_name, example_content_key(built.example))`, add: + +```python + if rules is not None and envelope.capture_id not in kept_from: + captured_at, recorded_model = capture_provenance(envelope) + reason = rules.stale_reason(parse_instant(captured_at), recorded_model) + if reason is not None: + ineligible[reason] += 1 + continue + if limits.max_cases is not None and envelope.capture_id in kept_from and not force: + skipped_existing += 1 + continue +``` + +(g) Right after the `case = PromotedCase(...)` construction and before its `try: write_promoted_case(...)`, add: + +```python + if limits.max_cases is not None and envelope.capture_id not in kept_from: + pending.append(case) + continue +``` + +(h) After the per-record loop's `if repaired_here: ...` block, still inside the suite loop, add: + +```python + line: str | None = None + if limits.max_cases is not None: + plan = plan_suite( + [ + KeptCase( + name=c.name, + instant=case_instant(c), + stale=stale_by_name.get(c.name), + reviewed=is_reviewed(c.example), + route=examples_route([c.example]), + content_key=example_content_key(c.example), + ) + for c in kept_cases + ], + _candidate_groups(pending), + max_cases=limits.max_cases, + ) + chosen = {cid for group in plan.add for cid in group.capture_ids} + for case in pending: + if case.name not in chosen: + continue + try: + write_promoted_case(case, base=base, force=force) + except FileExistsError: + skipped_existing += 1 + continue + promoted += 1 + if case.example.generation_config: + wired_generation += 1 + for name, _reason in plan.retire: + retire_promoted_case(suite_name, name, base=base) + if plan.reviewed_over_cap: + console.print( + f"[yellow]⚠[/yellow] {escape(suite_name)}: {plan.reviewed_over_cap} " + f"reviewed case(s) over captures.max_cases {limits.max_cases}; reviewed " + "cases are never retired, so it stays over.", + ) + line = _plan_line( + suite_name, + plan, + kept=len(kept_cases), + eligible=len(pending), + ineligible=ineligible, + max_age=limits.max_age, + ) + elif limits.active: + line = _plan_line( + suite_name, + None, + kept=len(kept_cases), + eligible=0, + ineligible=ineligible, + max_age=limits.max_age, + ) + if line is not None: + plan_lines.append(line) +``` + +(i) Right after `console.print(summary + ".")`: + +```python + for line in plan_lines: + console.print(escape(line), soft_wrap=True) +``` + +- [ ] **Step 5: Run the new tests and every capture test** + +Run: `uv run pytest tests/unit/test_cli_capture_limits.py tests/unit/test_cli_capture.py tests/unit/test_cli_capture_remote.py tests/unit/test_cli_capture_repair.py tests/unit/test_cli_capture_native_shapes.py tests/integration/test_capture_to_verdict.py -q` +Expected: PASS. If a `test_cli_capture_limits.py` count is off, debug with superpowers:systematic-debugging before touching the expectation — the expected strings are the spec's contract. + +- [ ] **Step 6: Static checks** + +Run: `uv run mypy --strict src/evalshift_cli && uv run ruff check . && uv run ruff format --check .` +Expected: clean. (`capture_sync` may trip ruff's complexity rules; if so, follow how the file already silences them for this function rather than adding a new suppression style.) + +- [ ] **Step 7: Commit** + +```bash +git add src/evalshift_cli/cli/commands/capture.py tests/unit/test_cli_capture_limits.py +git commit -m "feat(capture): sync keeps each suite to captures.max_cases fresh cases + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 9: Freshness in `doctor` and `compare` + +**Files:** +- Create: `src/evalshift_cli/captures/freshness.py` +- Modify: `src/evalshift_cli/cli/commands/doctor.py` (constants, `_suite_freshness_checks`, `run_checks`) +- Modify: `src/evalshift_cli/cli/commands/compare.py` (`_freshness_lines`, verdict block) +- Test: `tests/unit/test_captures_freshness.py` (new), `tests/unit/test_doctor.py`, `tests/unit/test_compare_command.py` + +**Interfaces:** +- Consumes: `case_instant` (Task 5), `model_filter_applies`, `same_model`, `describe_models` (Task 3), `format_duration` (Task 1). +- Produces: `FindingKind = Literal["unreadable", "stale_age", "stale_model", "no_source_model_case", "over_cap"]`; frozen dataclasses `FreshnessFinding(kind: FindingKind, message: str)`, `SuiteFreshness(suite: str, cases: int, newest: datetime | None, oldest: datetime | None, findings: tuple[FreshnessFinding, ...])` with `describe(now: datetime) -> str`; `suite_freshness(suite: str, suite_dir: Path, *, max_age: timedelta | None, max_cases: int | None, source_model: str | None, now: datetime) -> SuiteFreshness | None`; doctor `SUITE_FRESHNESS_CHECK = "suite freshness"`, `CAPTURES_LIMITS_CHECK = "captures limits"`; compare `_freshness_lines(cfg: EvalShiftConfig, suite_name: str | None, suite_path: Path, source_model: str) -> list[str]`. + +- [ ] **Step 1: Write the failing module tests** + +Create `tests/unit/test_captures_freshness.py`: + +```python +"""``suite_freshness``: what doctor and compare say about a committed capture suite.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from evalshift_cli.captures.freshness import SuiteFreshness, suite_freshness +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.promote import write_promoted_case +from evalshift_cli.suite.models import SuiteExample + +NOW = datetime(2026, 10, 16, 12, tzinfo=UTC) + + +def _write(base: Path, i: int, *, days: int, model: str = "claude-sonnet-5") -> None: + write_promoted_case( + PromotedCase( + name=f"cap_{i}", + suite="alpha", + from_capture=f"cap_{i}", + captured_at=(NOW - timedelta(days=days)).isoformat(), + source_model=model, + example=SuiteExample(id=f"cap_{i}", inputs={"input": f"q{i}"}, tools=[]), + ), + base=base, + ) + + +def _check(base: Path, **limits: Any) -> SuiteFreshness: + options: dict[str, Any] = { + "max_age": timedelta(days=14), + "max_cases": 5, + "source_model": "claude-sonnet-5", + **limits, + } + fresh = suite_freshness("alpha", base / "suites" / "alpha", now=NOW, **options) + assert fresh is not None + return fresh + + +def test_no_case_files_means_nothing_to_report(tmp_path: Path) -> None: + assert ( + suite_freshness( + "alpha", + tmp_path / "suites" / "alpha", + max_age=timedelta(days=14), + max_cases=5, + source_model="claude-sonnet-5", + now=NOW, + ) + is None + ) + + +def test_a_fresh_suite_has_no_findings(tmp_path: Path) -> None: + _write(tmp_path, 0, days=2) + _write(tmp_path, 1, days=12) + fresh = _check(tmp_path) + assert fresh.findings == () + assert fresh.describe(NOW) == "alpha: 2 cases, captured 2d–12d ago" + + +def test_a_suite_past_max_age_says_how_to_refresh(tmp_path: Path) -> None: + _write(tmp_path, 0, days=23) + [finding] = _check(tmp_path).findings + assert finding.kind == "stale_age" + assert finding.message == ( + "alpha: newest case captured 23d ago (captures.max_age 14d); " + "refresh with evalshift capture sync" + ) + + +def test_cases_on_another_model_are_counted(tmp_path: Path) -> None: + _write(tmp_path, 0, days=1) + _write(tmp_path, 1, days=1, model="claude-sonnet-4-5") + [finding] = _check(tmp_path).findings + assert finding.kind == "stale_model" + assert finding.message.startswith( + "alpha: 1 of 2 cases recorded on another model than claude-sonnet-5" + ) + + +def test_no_case_on_the_source_model_is_its_own_finding(tmp_path: Path) -> None: + _write(tmp_path, 0, days=1, model="gpt-5") + [finding] = _check(tmp_path).findings + assert finding.kind == "no_source_model_case" + assert "no case recorded on claude-sonnet-5 (seen: gpt-5 ×1)" in finding.message + + +def test_a_suite_over_its_cap_is_reported(tmp_path: Path) -> None: + for i in range(3): + _write(tmp_path, i, days=1) + [finding] = _check(tmp_path, max_cases=2).findings + assert finding.message == ( + "alpha: 3 cases, over captures.max_cases 2; evalshift capture sync trims it" + ) + + +def test_an_unreadable_case_file_is_a_finding_not_a_crash(tmp_path: Path) -> None: + _write(tmp_path, 0, days=1) + (tmp_path / "suites" / "alpha" / "broken.json").write_text("{", encoding="utf-8") + fresh = _check(tmp_path) + assert fresh.cases == 1 + assert [f.kind for f in fresh.findings] == ["unreadable"] + + +def test_a_legacy_case_ages_from_its_promotion(tmp_path: Path) -> None: + write_promoted_case( + PromotedCase( + name="old", + suite="alpha", + from_capture="old", + promoted_at=(NOW - timedelta(days=30)).isoformat(), + example=SuiteExample(id="old", inputs={"input": "q"}, tools=[]), + ), + base=tmp_path, + ) + assert [f.kind for f in _check(tmp_path).findings] == ["stale_age"] +``` + + +- [ ] **Step 2: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_captures_freshness.py -q` +Expected: FAIL — `ModuleNotFoundError: No module named 'evalshift_cli.captures.freshness'` + +- [ ] **Step 3: Implement `captures/freshness.py`** + +```python +"""How fresh a committed capture suite is, for ``doctor`` and ``compare`` (spec §7). + +Reads the promoted case files committed beside ``golden.jsonl`` -- the only place a case's +capture time and model live -- so a team that only runs CI still learns its suite went stale. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from pathlib import Path +from typing import Literal + +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.staleness import case_instant +from evalshift_cli.models.matching import describe_models, model_filter_applies, same_model +from evalshift_cli.utils.duration import format_duration + +FindingKind = Literal["unreadable", "stale_age", "stale_model", "no_source_model_case", "over_cap"] + + +@dataclass(frozen=True, slots=True) +class FreshnessFinding: + """One thing that is stale about a suite, worded for a doctor row or a compare line.""" + + kind: FindingKind + message: str + + +@dataclass(frozen=True, slots=True) +class SuiteFreshness: + """A suite's case count, capture-time span and findings.""" + + suite: str + cases: int + newest: datetime | None + oldest: datetime | None + findings: tuple[FreshnessFinding, ...] + + def describe(self, now: datetime) -> str: + """``alpha: 200 cases, captured 2d–12d ago`` (the span left out when unknown).""" + text = f"{self.suite}: {self.cases} cases" + if self.newest is not None and self.oldest is not None: + text += f", captured {_age(now, self.newest)}–{_age(now, self.oldest)} ago" + return text + + +def _age(now: datetime, then: datetime) -> str: + return f"{max(0, (now - then).days)}d" + + +def suite_freshness( + suite: str, + suite_dir: Path, + *, + max_age: timedelta | None, + max_cases: int | None, + source_model: str | None, + now: datetime, +) -> SuiteFreshness | None: + """Read ``suite_dir``'s case files and report what is stale; ``None`` when there are none. + + Never raises: an unreadable case file becomes one ``unreadable`` finding and is left out + of every count. + """ + paths = sorted(suite_dir.glob("*.json")) if suite_dir.is_dir() else [] + cases: list[PromotedCase] = [] + unreadable = 0 + for path in paths: + try: + cases.append(PromotedCase.model_validate_json(path.read_text(encoding="utf-8"))) + except (OSError, ValueError): # ValidationError and UnicodeDecodeError are ValueErrors + unreadable += 1 + if not cases and not unreadable: + return None + instants = [i for i in (case_instant(c) for c in cases) if i is not None] + newest = max(instants, default=None) + oldest = min(instants, default=None) + findings: list[FreshnessFinding] = [] + if unreadable: + findings.append( + FreshnessFinding( + "unreadable", + f"{suite}: {unreadable} case file(s) could not be read; " + "the counts leave them out", + ) + ) + if max_age is not None and newest is not None and newest < now - max_age: + findings.append( + FreshnessFinding( + "stale_age", + f"{suite}: newest case captured {_age(now, newest)} ago " + f"(captures.max_age {format_duration(max_age)}); " + "refresh with evalshift capture sync", + ) + ) + models = [c.source_model for c in cases] + if source_model and any(models): + if model_filter_applies(models, source_model): + other = sum(1 for m in models if m and not same_model(m, source_model)) + if other: + findings.append( + FreshnessFinding( + "stale_model", + f"{suite}: {other} of {len(cases)} cases recorded on another model " + f"than {source_model}; evalshift capture sync replaces them as new " + "captures arrive", + ) + ) + else: + findings.append( + FreshnessFinding( + "no_source_model_case", + f"{suite}: no case recorded on {source_model} " + f"(seen: {describe_models(models)}); check defaults.source_model, " + "or record new captures", + ) + ) + if max_cases is not None and len(cases) > max_cases: + findings.append( + FreshnessFinding( + "over_cap", + f"{suite}: {len(cases)} cases, over captures.max_cases {max_cases}; " + "evalshift capture sync trims it", + ) + ) + return SuiteFreshness( + suite=suite, cases=len(cases), newest=newest, oldest=oldest, findings=tuple(findings) + ) +``` + +Run: `uv run pytest tests/unit/test_captures_freshness.py -q` — Expected: PASS. + +- [ ] **Step 4: Write the failing doctor tests** + +Append to `tests/unit/test_doctor.py` (imports: `from datetime import UTC, datetime, timedelta`; `PromotedCase`; `rebuild_golden_jsonl`, `write_promoted_case` from `evalshift_cli.captures.promote`; `SuiteExample`; `CAPTURES_LIMITS_CHECK`, `SUITE_FRESHNESS_CHECK` from the doctor module — skip any already imported): + +```python +class TestSuiteFreshnessCheck: + def _project( + self, tmp_path: Path, *, captures: str, ages_days: list[int], model: str = "claude-sonnet-5" + ) -> None: + (tmp_path / CONFIG_FILENAME).write_text( + "prompts:\n - id: p\n detection: manual\n content: '{input}'\n" + " variables: [input]\n" + "defaults:\n source_model: claude-sonnet-5\n" + f"{captures}" + "suites:\n alpha:\n source: captured\n" + " path: .evalshift/suites/alpha/golden.jsonl\n", + encoding="utf-8", + ) + now = datetime.now(UTC) + for i, days in enumerate(ages_days): + write_promoted_case( + PromotedCase( + name=f"cap_{i}", + suite="alpha", + from_capture=f"cap_{i}", + captured_at=(now - timedelta(days=days)).isoformat(), + source_model=model, + example=SuiteExample(id=f"cap_{i}", inputs={"input": f"q{i}"}, tools=[]), + ), + base=tmp_path / ".evalshift", + ) + rebuild_golden_jsonl(tmp_path / ".evalshift" / "suites" / "alpha") + + def _rows(self, tmp_path: Path, name: str = SUITE_FRESHNESS_CHECK) -> list[CheckResult]: + return [r for r in run_checks(cwd=tmp_path, env=_empty_env()) if r.name == name] + + def test_a_fresh_suite_is_ok(self, tmp_path: Path) -> None: + self._project(tmp_path, captures="captures:\n max_age: 14d\n max_cases: 5\n", ages_days=[2, 12]) + [row] = self._rows(tmp_path) + assert row.status == "ok" + assert row.detail == "alpha: 2 cases, captured 2d–12d ago" + + def test_a_stale_suite_warns(self, tmp_path: Path) -> None: + self._project(tmp_path, captures="captures:\n max_age: 14d\n", ages_days=[23, 30]) + [row] = self._rows(tmp_path) + assert row.status == "warn" + assert "newest case captured 23d ago (captures.max_age 14d)" in row.detail + + def test_no_rows_without_limits(self, tmp_path: Path) -> None: + self._project(tmp_path, captures="", ages_days=[30]) + assert self._rows(tmp_path) == [] + assert self._rows(tmp_path, CAPTURES_LIMITS_CHECK) == [] + + def test_a_bucket_without_limits_gets_a_nudge( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from tests.unit.fake_remote_store import populated_store + + self._project(tmp_path, captures="captures:\n store: s3://acme-evals/p\n", ages_days=[1]) + monkeypatch.setattr(doctor_module, "open_store", lambda parsed: populated_store("cap_1")) + [row] = self._rows(tmp_path, CAPTURES_LIMITS_CHECK) + assert row.status == "warn" + assert row.detail == ( + "suites grow with every capture sync; set captures.max_cases and captures.max_age" + ) +``` + +(If `CheckResult` is not yet imported in the test module, import it from the doctor module.) + +- [ ] **Step 5: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_doctor.py -q -k SuiteFreshness` +Expected: FAIL — `ImportError` for the two constants. + +- [ ] **Step 6: Implement the doctor rows** + +In `cli/commands/doctor.py` add `from datetime import UTC, datetime` and `from evalshift_cli.captures.freshness import suite_freshness`; next to the other row-name constants: + +```python +SUITE_FRESHNESS_CHECK: Final = "suite freshness" +CAPTURES_LIMITS_CHECK: Final = "captures limits" +``` + +below `_captures_store_check`: + +```python +def _suite_freshness_checks(cwd: Path) -> list[CheckResult]: + """Bounded-suite rows (spec §7). Never ``fail``: freshness is advice. + + ``captures limits`` warns when ``captures.store`` is set without ``max_age`` / + ``max_cases`` -- every sync then adds to the suite. With a limit set, each + ``source: captured`` suite gets one ``ok`` row, or one ``warn`` row per finding. + """ + cfg_path = cwd / CONFIG_FILENAME + if not cfg_path.exists(): + return [] + try: + cfg = load_config(cfg_path) + except ConfigError: + return [] + captures = cfg.captures + if captures.max_age is None and captures.max_cases is None: + if captures.store is None: + return [] + return [ + CheckResult( + name=CAPTURES_LIMITS_CHECK, + status="warn", + detail="suites grow with every capture sync; " + "set captures.max_cases and captures.max_age", + ) + ] + now = datetime.now(UTC) + rows: list[CheckResult] = [] + for name, path in _named_suite_paths(cwd, cfg): + entry = cfg.suites.get(name) + if entry is None or entry.source != "captured": + continue + fresh = suite_freshness( + name, + path.parent, + max_age=captures.max_age, + max_cases=captures.max_cases, + source_model=cfg.defaults.source_model, + now=now, + ) + if fresh is None: + continue + if not fresh.findings: + rows.append( + CheckResult(name=SUITE_FRESHNESS_CHECK, status="ok", detail=fresh.describe(now)) + ) + rows.extend( + CheckResult(name=SUITE_FRESHNESS_CHECK, status="warn", detail=f.message) + for f in fresh.findings + ) + return rows +``` + +and in `run_checks`, right after `results.extend(_captures_store_check(cwd))`: + +```python + results.extend(_suite_freshness_checks(cwd)) +``` + +Run: `uv run pytest tests/unit/test_doctor.py -q` — Expected: PASS (fix any test that pins the full row list). + +- [ ] **Step 7: Write the failing compare tests** + +Append to `tests/unit/test_compare_command.py` (imports: `PromotedCase`, `rebuild_golden_jsonl`, `write_promoted_case`, `SuiteExample`, `EvalShiftConfig`, and `_freshness_lines` from `evalshift_cli.cli.commands.compare` — skip any already imported): + +```python +def _captured_cases(tmp_path: Path, suite: str, n: int) -> Path: + for i in range(n): + write_promoted_case( + PromotedCase( + name=f"ex{i}", + suite=suite, + from_capture=f"cap_{i}", + example=SuiteExample(id=f"ex{i}", inputs={"name": f"User{i}"}, tools=[]), + ), + base=tmp_path / ".evalshift", + ) + return rebuild_golden_jsonl(tmp_path / ".evalshift" / "suites" / suite) + + +class TestFreshnessLines: + def _cfg(self, captures: dict[str, object]) -> EvalShiftConfig: + return EvalShiftConfig( + prompts=[{"id": "p", "detection": "manual", "content": "{name}", "variables": ["name"]}], + captures=captures, + suites={"greet": {"source": "captured", "path": ".evalshift/suites/greet/golden.jsonl"}}, + ) + + def test_an_over_cap_suite_gets_a_line(self, tmp_path: Path) -> None: + golden = _captured_cases(tmp_path, "greet", 3) + [line] = _freshness_lines(self._cfg({"max_cases": 2}), "greet", golden, "gemini-2.5-flash") + assert "greet: 3 cases, over captures.max_cases 2" in line + + def test_no_lines_without_limits(self, tmp_path: Path) -> None: + golden = _captured_cases(tmp_path, "greet", 3) + assert _freshness_lines(self._cfg({}), "greet", golden, "gemini-2.5-flash") == [] + + def test_no_lines_for_a_suite_given_by_path(self, tmp_path: Path) -> None: + golden = _captured_cases(tmp_path, "greet", 3) + assert _freshness_lines(self._cfg({"max_cases": 2}), None, golden, "gemini-2.5-flash") == [] +``` + +and inside the end-to-end test class that holds `test_all_prints_the_tool_free_warning_it_scored_quietly`: + +```python + def test_compare_warns_when_the_suite_is_over_its_cap( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + _scaffold(tmp_path) + config_path = tmp_path / "evalshift.yaml" + config_path.write_text( + config_path.read_text(encoding="utf-8") + + " captures:\n max_cases: 2\n" + + " suites:\n greet:\n source: captured\n" + + " path: .evalshift/suites/greet/golden.jsonl\n", + encoding="utf-8", + ) + _captured_cases(tmp_path, "greet", 3) + _patch_client(monkeypatch) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(app, ["compare", "--suite-name", "greet", "--yes"]) + + assert result.exit_code == 0, result.output + flat = " ".join(result.output.split()) + assert "greet: 3 cases, over captures.max_cases 2; evalshift capture sync trims it" in flat + assert flat.count("over captures.max_cases") == 1 +``` + +- [ ] **Step 8: Run them to see them fail** + +Run: `uv run pytest tests/unit/test_compare_command.py -q -k "Freshness or over_its_cap"` +Expected: FAIL — `ImportError: cannot import name '_freshness_lines'` + +- [ ] **Step 9: Implement the compare lines** + +In `cli/commands/compare.py` add `from datetime import UTC, datetime`, `from rich.markup import escape`, `EvalShiftConfig` to the `evalshift_cli.config.models` import, and `from evalshift_cli.captures.freshness import suite_freshness`. Add above `compare_command`: + +```python +def _freshness_lines( + cfg: EvalShiftConfig, suite_name: str | None, suite_path: Path, source_model: str +) -> list[str]: + """``⚠`` lines for a stale captured suite (spec §7). + + compare drops doctor's ``warn`` rows, so it asks for the findings itself. Only for a suite + selected by name whose entry is ``source: captured``, and only with a capture limit set. + """ + captures = cfg.captures + if suite_name is None or (captures.max_age is None and captures.max_cases is None): + return [] + entry = cfg.suites.get(suite_name) + if entry is None or entry.source != "captured": + return [] + fresh = suite_freshness( + suite_name, + suite_path.parent, + max_age=captures.max_age, + max_cases=captures.max_cases, + source_model=source_model, + now=datetime.now(UTC), + ) + if fresh is None: + return [] + return [f"[yellow]⚠[/yellow] {escape(f.message)}" for f in fresh.findings] +``` + +In `compare_command`, right after the `suite = load_jsonl(suite_path)` try/except: + +```python + freshness = _freshness_lines(cfg, suite_name, suite_path, source_model) +``` + +and in the verdict block, right after the `tool_free_warnings` block: + +```python + # And again: a stale suite gates on ground truth production no longer produces. + if freshness: + for line in freshness: + console.print(line, soft_wrap=True) + console.print() +``` + +- [ ] **Step 10: Run doctor, compare and freshness tests, then static checks** + +Run: `uv run pytest tests/unit/test_captures_freshness.py tests/unit/test_doctor.py tests/unit/test_compare_command.py -q && uv run mypy --strict src/evalshift_cli && uv run ruff check . && uv run ruff format --check .` +Expected: PASS / clean. + +- [ ] **Step 11: Commit** + +```bash +git add src/evalshift_cli/captures/freshness.py src/evalshift_cli/cli/commands/doctor.py src/evalshift_cli/cli/commands/compare.py tests/unit/test_captures_freshness.py tests/unit/test_doctor.py tests/unit/test_compare_command.py +git commit -m "feat(doctor,compare): warn when a captured suite is stale or over its cap + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 10: Lifecycle integration test + +**Files:** +- Test: `tests/integration/test_capture_lifecycle.py` (new) + +**Interfaces:** +- Consumes: everything above, through the CLI only; `FakeRemoteStore`, `capture_payload`, `TOOLSET_KEY`, `TOOLSET_PAYLOAD` from `tests/unit/fake_remote_store.py`. + +- [ ] **Step 1: Write the test** + +Create `tests/integration/test_capture_lifecycle.py`: + +```python +"""Weeks of production captures in a bucket stay one bounded, fresh suite (spec, Testing).""" + +from __future__ import annotations + +import re +import shutil +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from evalshift_cli.captures.store_uri import StoreURI +from evalshift_cli.cli.commands import capture as capture_module +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config +from evalshift_cli.cli.main import app +from tests.unit.fake_remote_store import ( + TOOLSET_KEY, + TOOLSET_PAYLOAD, + FakeRemoteStore, + capture_payload, +) + +runner = CliRunner() + + +def _day(day: int, month: int = 6) -> datetime: + return datetime(2026, month, day, 12, tzinfo=UTC) + + +class _Bucket: + """A fake bucket the SDK 'writes' to between syncs.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.store = FakeRemoteStore({TOOLSET_KEY: TOOLSET_PAYLOAD}) + monkeypatch.setattr(capture_module, "open_store", lambda parsed: self._open(parsed)) + + def _open(self, parsed: StoreURI) -> FakeRemoteStore: + return self.store + + def record(self, cid: str, when: datetime, *, model: str = "claude-sonnet-5-20260301") -> None: + key = f"captures/planner/{cid}.json" + self.store.objects[key] = capture_payload( + cid, + "planner", + model_input=f"plan my day {cid}", + created_at=when.isoformat(), + model_id=model, + ) + self.store.modified[key] = when + + +def _config(path: Path, *, source_model: str = "claude-sonnet-5") -> None: + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + text = re.sub(r"(?m)^ source_model: .*$", f" source_model: {source_model}", text) + text += "\ncaptures:\n store: s3://acme-evals/testapp\n max_age: 14d\n max_cases: 4\n" + path.write_text(text, encoding="utf-8") + + +def _sync(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, now: datetime) -> Any: + monkeypatch.setattr(capture_module, "_now", lambda: now) + result = runner.invoke( + app, + ["capture", "sync", "--config", str(tmp_path / "evalshift.yaml"), "--base", str(tmp_path)], + ) + assert result.exit_code == 0, result.stdout + return result + + +def _cases(tmp_path: Path) -> set[str]: + return {p.stem for p in (tmp_path / "suites" / "planner").glob("*.json")} + + +def test_weeks_of_captures_stay_one_bounded_fresh_suite( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml") + + # Week 1: six captures, a suite of four. + for i in range(6): + bucket.record(f"w1_{i}", _day(10)) + _sync(tmp_path, monkeypatch, _day(12)) + week_1 = _cases(tmp_path) + assert len(week_1) == 4 + + # Week 2, inside max_age: more captures, but nothing is stale -- the suite holds still. + for i in range(6): + bucket.record(f"w2_{i}", _day(17)) + _sync(tmp_path, monkeypatch, _day(19)) + assert _cases(tmp_path) == week_1 + + # Two weeks later: week-1 cases are stale and fresh captures replace them. + for i in range(3): + bucket.record(f"w4_{i}", _day(1, month=7)) + _sync(tmp_path, monkeypatch, _day(2, month=7)) + cases = _cases(tmp_path) + assert len(cases) == 4 + assert {f"w4_{i}" for i in range(3)} <= cases + + # A fresh checkout: suites committed, local mirror gone. Promoted captures are not + # downloaded again, and nothing outside max_age is. + shutil.rmtree(tmp_path / "captures") + bucket.store.gets.clear() + result = _sync(tmp_path, monkeypatch, _day(2, month=7)) + downloaded = {key.rsplit("/", 1)[-1].removesuffix(".json") for key in bucket.store.gets} + assert not downloaded & cases + assert not any(cid.startswith(("w1_", "w2_")) for cid in downloaded) + assert _cases(tmp_path) == cases + assert "promoted" in result.stdout + + +def test_a_production_migration_retires_the_old_models_cases( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml") + for i in range(4): + bucket.record(f"old_{i}", _day(10)) + _sync(tmp_path, monkeypatch, _day(12)) + + _config(tmp_path / "evalshift.yaml", source_model="claude-sonnet-5-5") + for i in range(2): + bucket.record(f"new_{i}", _day(13), model="claude-sonnet-5-5-20260601") + result = _sync(tmp_path, monkeypatch, _day(14)) + + assert {"new_0", "new_1"} <= _cases(tmp_path) + assert len(_cases(tmp_path)) == 4 + assert "retired 2 (2 on another model)" in " ".join(result.stdout.split()) + + +def test_a_wrong_source_model_trips_the_guardrail( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml", source_model="gpt-9") + for i in range(3): + bucket.record(f"cap_{i}", _day(10)) + + result = _sync(tmp_path, monkeypatch, _day(12)) + + assert len(_cases(tmp_path)) == 3 + assert "not filtering by model — check defaults.source_model" in " ".join( + result.stdout.split() + ) +``` + +- [ ] **Step 2: Run it** + +Run: `uv run pytest tests/integration/test_capture_lifecycle.py -q` +Expected: PASS. A failure here is a real integration bug in Tasks 7–8: debug it (superpowers:systematic-debugging) rather than loosening the assertions. + +- [ ] **Step 3: Commit** + +```bash +git add tests/integration/test_capture_lifecycle.py +git commit -m "test(capture): weeks of bucket captures stay one bounded fresh suite + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 11: Docs, changelog, spec and plan + +**Files:** +- Modify: `CHANGELOG.md`, `DOCS.md`, `llms-full.txt`, `docs/configuration.md`, `docs/sdk.md`, `docs/conversations.md` +- Add (forced, gitignored): `docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md`, `docs/superpowers/plans/2026-10-16-bounded-capture-suites.md` + +- [ ] **Step 1: CHANGELOG** + +Under `## [Unreleased]` in `CHANGELOG.md`: + +```markdown +### Added + +- **Bounded capture suites.** Two optional settings under `captures:` — + `max_age: 14d` and `max_cases: 200` — keep each captured suite a capped, + route-balanced sample of recent production on `defaults.source_model`. + `capture sync` only promotes captures recorded within `max_age` on the source + model, never grows a suite past `max_cases`, and swaps stale cases (too old, or + recorded on another model) for fresh captures as they arrive — never shrinking + a suite that has nothing to replace them with. A conversation enters whole or + not at all; cases a human marked `provenance: reviewed` are never retired; the + same captures give the same suite on any machine. `max_age` also bounds what + `sync` downloads from `captures.store` (`--since` still overrides it). Sync + prints one line per suite: kept / retired / added, and what was not eligible. + If no capture was recorded on `defaults.source_model`, sync warns and does not + filter by model rather than empty the suite. +- `doctor` gains a `suite freshness` row per captured suite (newest case older + than `max_age`, cases on another model, over `max_cases`) and a + `captures limits` warning when `captures.store` is set without limits; + `compare` prints the same findings above the verdict, so a CI-only team sees + its suite went stale. +- Promoted case files record `captured_at` and `source_model`; `capture sync` + backfills them on existing cases whose capture is still local. + +### Changed + +- `evalshift init` scaffolds `captures: {max_age: 14d, max_cases: 200}` (with a + commented `store:`). Existing configs are untouched — without the two keys, + `capture sync` behaves exactly as before. +- `capture sync` now reads `evalshift.yaml` under `--offline` too, for the limits. +- Older CLIs reject the new `captures` keys and the new case-file fields: bump + the CLI everywhere `capture` commands run (and a pinned CI action's + `evalshift-version`) before adding them. CI's `compare` reads only + `golden.jsonl`, whose format is unchanged. +``` + +- [ ] **Step 2: Reference docs** + +- `docs/configuration.md` `## captures` (line ~1066): add `max_age` and `max_cases` rows to the field table (types, defaults `None`, the one-sentence meaning from the `CapturesConfig` docstring); add a subsection **Bounded suites** that states the opt-in rule, eligibility (§3), the plan (§5 rules 1–4, in prose), the guardrail, the per-suite line format with an example, and the fetch window; extend the bucket-retention paragraph (line ~1104) with an expiration rule for the `captures/` prefix only (not `toolsets/`, which committed suites reference): + + ```bash + aws s3api put-bucket-lifecycle-configuration --bucket acme-evals \ + --lifecycle-configuration '{"Rules": [{"ID": "expire-evalshift-captures", + "Status": "Enabled", "Filter": {"Prefix": "support-agent/captures/"}, + "Expiration": {"Days": 30}}]}' + ``` + + plus one line each for GCS (Object Lifecycle Management `age` condition) and Azure (lifecycle management `daysAfterModificationGreaterThan`), and the advice "expire after a period ≥ `max_age`". +- `DOCS.md`: `## Capturing from production` — extend the `captures:` YAML example with `max_age` / `max_cases` and add a short "Bounded suites" paragraph linking to `docs/configuration.md#captures`; the sync steps list gains a step "With `max_age` / `max_cases` set: filter, plan, retire" (one paragraph); `### Top-level fields` `captures` row names the two keys; the doctor bullet under `## How it works` names the `suite freshness` and `captures limits` rows; `## Command reference` sync/doctor/compare entries mention them; `evalshift init` description says it scaffolds the limits. +- `llms-full.txt`: mirror every DOCS.md change in its matching section (init 137-158, doctor 162-198, compare 235-239, captures 327-420, and the `captures:` schema block ~801-811). +- `docs/sdk.md` §2: after "Bucket retention is a lifecycle rule", add "— and `captures.max_age` / `captures.max_cases` in `evalshift.yaml` keep the suites built from it bounded (see [configuration](configuration.md#captures))."; §3: show the two keys in the `captures:` example. +- `docs/conversations.md`: one sentence — with `captures.max_cases` set, a conversation's new turns enter a suite together or not at all. + +- [ ] **Step 3: Docs tests** + +Run: `uv run pytest tests/unit/test_docs_currency.py tests/unit/test_init.py -q` +Expected: PASS. + +- [ ] **Step 4: Commit (spec and plan forced in)** + +```bash +git add CHANGELOG.md DOCS.md llms-full.txt docs/configuration.md docs/sdk.md docs/conversations.md +git add -f docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md docs/superpowers/plans/2026-10-16-bounded-capture-suites.md +git commit -m "docs: bounded capture suites (captures.max_age / captures.max_cases) + +Co-Authored-By: Claude Opus 5.5 " +``` + +--- + +### Task 12: Verify and open the PR + +- [ ] **Step 1: Full gate** + +Run: `make ci` then `pre-commit run --all-files` +Expected: both green; show the output (superpowers:verification-before-completion). + +- [ ] **Step 2: Real-project smoke (read-only on the bucket)** + +In `/home/lukas/repos/testApp` with this branch's CLI (`uv run --project /path/to/worktree evalshift ...`), on a scratch copy of its `evalshift.yaml` with `max_age: 14d` / `max_cases: 200` added: `evalshift capture sync --print` then `evalshift doctor`. Confirm the per-suite lines and freshness rows read sensibly. Do not commit anything in testApp; restore its config. + +- [ ] **Step 3: Code review** + +Use superpowers:requesting-code-review on the branch (reviewer on `model: opus`), with the spec and this plan as context and the Review Focus list as the checklist. Address findings via superpowers:receiving-code-review. + +- [ ] **Step 4: Push and open the PR** + +```bash +git push -u origin feat/bounded-capture-suites +gh pr create --title "feat(capture): bounded capture suites (captures.max_age / captures.max_cases)" --body " + +🤖 Generated with [Claude Code](https://claude.com/claude-code)" +``` + +The maintainer merges the PR; the release (version bump, tag) is a separate PR. From d1cef86d9b0226d6cce505f33ce08433a45a4f46 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:01:48 +0200 Subject: [PATCH 03/26] feat(config): captures.max_age and captures.max_cases Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/remote.py | 13 +++++---- src/evalshift_cli/config/models.py | 34 +++++++++++++++++++++- src/evalshift_cli/utils/duration.py | 36 ++++++++++++++++++++++++ tests/unit/test_config_captures.py | 39 ++++++++++++++++++++++++++ tests/unit/test_utils_duration.py | 42 ++++++++++++++++++++++++++++ 5 files changed, 157 insertions(+), 7 deletions(-) create mode 100644 src/evalshift_cli/utils/duration.py create mode 100644 tests/unit/test_utils_duration.py diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py index 9f20931..1cbc371 100644 --- a/src/evalshift_cli/captures/remote.py +++ b/src/evalshift_cli/captures/remote.py @@ -40,6 +40,7 @@ from evalshift_cli.captures.reader import captures_root, toolsets_root from evalshift_cli.captures.store_uri import StoreURI, package_for_module +from evalshift_cli.utils.duration import parse_duration #: Largest object a fetch will download. Captures are kilobytes; this is a safety stop. MAX_OBJECT_BYTES = 32 * 1024 * 1024 @@ -47,7 +48,6 @@ # Always used with ``fullmatch``: ``$`` also matches before a trailing newline. _CAPTURE_NAME = re.compile(r"[A-Za-z0-9_-]+\.json") _TOOLSET_NAME = re.compile(r"[0-9a-f]{64}\.json") -_DURATION = re.compile(r"(\d+)([mhd])") #: How to fix credentials, per store scheme, for a list or download failure. _CREDENTIALS_HINT: dict[str, str] = { @@ -230,12 +230,13 @@ def parse_since(text: str, *, now: datetime | None = None) -> datetime: """ reference = now if now is not None else datetime.now(UTC) cleaned = text.strip() - match = _DURATION.fullmatch(cleaned) try: - if match: - amount, unit = int(match.group(1)), match.group(2) - delta = {"m": timedelta(minutes=1), "h": timedelta(hours=1), "d": timedelta(days=1)} - return reference - delta[unit] * amount + delta: timedelta | None = parse_duration(cleaned) + except ValueError: + delta = None + try: + if delta is not None: + return reference - delta parsed = datetime.fromisoformat(cleaned) except (ValueError, OverflowError) as exc: # OverflowError: a duration reaching before year 1 raise ValueError( diff --git a/src/evalshift_cli/config/models.py b/src/evalshift_cli/config/models.py index 3953c5c..063dcca 100644 --- a/src/evalshift_cli/config/models.py +++ b/src/evalshift_cli/config/models.py @@ -15,14 +15,23 @@ from __future__ import annotations +from datetime import timedelta from typing import Annotated, Any, Literal, Self from jsonschema import Draft202012Validator from jsonschema.exceptions import SchemaError -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import ( + BaseModel, + BeforeValidator, + ConfigDict, + Field, + field_validator, + model_validator, +) from evalshift_cli.captures.store_uri import STORE_URI_FORMS, parse_store_uri from evalshift_cli.suite.tags import RESERVED_SLICE_NAME +from evalshift_cli.utils.duration import DURATION_FORMS, parse_duration # Default judge model used across the config when none is specified. # Centralised here so a single edit propagates everywhere. Gemini's @@ -816,6 +825,15 @@ class Retention(_StrictModel): run_ttl_days: int | None = Field(default=None, ge=1) +def _duration_from_yaml(value: object) -> object: + """``captures.max_age`` is written ``14d``; a bare number would silently mean seconds.""" + if value is None or isinstance(value, timedelta): + return value + if not isinstance(value, str): + raise ValueError(f"expected a duration like {DURATION_FORMS}") + return parse_duration(value) + + class CapturesConfig(_StrictModel): """Where the capture SDK's recordings live, beyond the local ``.evalshift/`` directory. @@ -830,9 +848,23 @@ class CapturesConfig(_StrictModel): ``az:////``. Credentials come from the provider's default chain, never from this file; a URI carrying ``@`` or ``?`` is rejected. ``None`` (the default) means local disk only. + max_age: Captures recorded longer ago than this (``30m``, ``24h``, ``14d``) never enter a + suite, and promoted cases that old are stale: ``capture sync`` replaces them as fresh + captures arrive. Also the fetch window. ``None`` (the default): no age limit. + max_cases: The most cases ``capture sync`` keeps in each captured suite. ``None`` (the + default): suites grow with every sync. Setting either limit turns bounded suites on. """ store: str | None = None + max_age: Annotated[timedelta | None, BeforeValidator(_duration_from_yaml)] = None + max_cases: int | None = Field(default=None, ge=1) + + @field_validator("max_age") + @classmethod + def _positive_max_age(cls, value: timedelta | None) -> timedelta | None: + if value is not None and value <= timedelta(0): + raise ValueError(f"must be greater than 0 (expected a duration like {DURATION_FORMS})") + return value @field_validator("store") @classmethod diff --git a/src/evalshift_cli/utils/duration.py b/src/evalshift_cli/utils/duration.py new file mode 100644 index 0000000..064ad8d --- /dev/null +++ b/src/evalshift_cli/utils/duration.py @@ -0,0 +1,36 @@ +"""The duration grammar shared by ``--since`` and ``captures.max_age``: ``N`` + ``m|h|d``.""" + +from __future__ import annotations + +import re +from datetime import timedelta + +#: How an error names the accepted forms. +DURATION_FORMS = "30m, 24h or 14d" + +_DURATION = re.compile(r"(\d+)([mhd])") +_UNITS = {"m": timedelta(minutes=1), "h": timedelta(hours=1), "d": timedelta(days=1)} + + +def parse_duration(text: str) -> timedelta: + """Parse ``30m`` / ``24h`` / ``14d`` (surrounding whitespace allowed) into a timedelta. + + Raises: + ValueError: when ``text`` is not that form, or is too large for a timedelta. + """ + match = _DURATION.fullmatch(text.strip()) + if match is None: + raise ValueError(f"expected a duration like {DURATION_FORMS}, got {text!r}") + try: + return _UNITS[match.group(2)] * int(match.group(1)) + except OverflowError as exc: + raise ValueError(f"duration {text!r} is too large") from exc + + +def format_duration(delta: timedelta) -> str: + """Render ``delta`` in the largest whole unit :func:`parse_duration` reads (``14d``, ``36h``).""" + seconds = int(delta.total_seconds()) + for suffix, unit in (("d", 86400), ("h", 3600), ("m", 60)): + if seconds % unit == 0: + return f"{seconds // unit}{suffix}" + return f"{seconds}s" diff --git a/tests/unit/test_config_captures.py b/tests/unit/test_config_captures.py index e9474a1..13f18a2 100644 --- a/tests/unit/test_config_captures.py +++ b/tests/unit/test_config_captures.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import timedelta from pathlib import Path import pytest @@ -96,3 +97,41 @@ def test_loader_error_never_echoes_a_secret_store( for secret in secrets: assert secret not in plain assert secret not in rendered + + +def test_limits_default_to_off() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT]) + assert cfg.captures.max_age is None + assert cfg.captures.max_cases is None + + +def test_limits_parse_without_a_store() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": "14d", "max_cases": 200}) + assert cfg.captures.max_age == timedelta(days=14) + assert cfg.captures.max_cases == 200 + assert cfg.captures.store is None + + +@pytest.mark.parametrize("value", ["2w", "14", 14, "0d", "P14D"]) +def test_max_age_rejects_anything_but_a_positive_duration(value: object) -> None: + with pytest.raises(ValidationError, match="max_age"): + EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": value}) + + +@pytest.mark.parametrize("value", [0, -5]) +def test_max_cases_must_be_positive(value: int) -> None: + with pytest.raises(ValidationError, match="max_cases"): + EvalShiftConfig(prompts=[_PROMPT], captures={"max_cases": value}) + + +def test_loader_reports_max_age_error_at_its_location(tmp_path: Path) -> None: + path = tmp_path / "evalshift.yaml" + path.write_text( + "prompts:\n - id: p\n detection: manual\n content: hi\ncaptures:\n max_age: 2w\n", + encoding="utf-8", + ) + with pytest.raises(ConfigError) as info: + load_config(path) + [detail] = info.value.details + assert detail.location == "captures.max_age" + assert "30m, 24h or 14d" in detail.message diff --git a/tests/unit/test_utils_duration.py b/tests/unit/test_utils_duration.py new file mode 100644 index 0000000..799996f --- /dev/null +++ b/tests/unit/test_utils_duration.py @@ -0,0 +1,42 @@ +"""The duration grammar shared by ``--since`` and ``captures.max_age``.""" + +from __future__ import annotations + +from datetime import timedelta + +import pytest + +from evalshift_cli.utils.duration import format_duration, parse_duration + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("30m", timedelta(minutes=30)), + ("24h", timedelta(hours=24)), + ("14d", timedelta(days=14)), + (" 7d ", timedelta(days=7)), + ("0d", timedelta(0)), + ], +) +def test_parse_duration_accepts_minutes_hours_days(text: str, expected: timedelta) -> None: + assert parse_duration(text) == expected + + +@pytest.mark.parametrize("text", ["", "14", "2w", "1.5d", "-1d", "d", "14 d", "P14D"]) +def test_parse_duration_rejects_anything_else(text: str) -> None: + with pytest.raises(ValueError, match="expected a duration like 30m, 24h or 14d"): + parse_duration(text) + + +def test_parse_duration_overflow_is_a_value_error() -> None: + with pytest.raises(ValueError, match="too large"): + parse_duration("9999999999999d") + + +@pytest.mark.parametrize( + ("delta", "text"), + [(timedelta(days=14), "14d"), (timedelta(hours=36), "36h"), (timedelta(minutes=90), "90m")], +) +def test_format_duration_uses_the_largest_whole_unit(delta: timedelta, text: str) -> None: + assert format_duration(delta) == text From c8512637e36ca8ccc74f2117a54479be3377ac05 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:07:23 +0200 Subject: [PATCH 04/26] feat(init): scaffold captures.max_age and captures.max_cases Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/init.py | 26 +++++++++++++++++++++++++- tests/unit/test_cli_capture.py | 11 ++++++++--- tests/unit/test_cli_capture_remote.py | 10 +++++----- tests/unit/test_cli_capture_repair.py | 5 +++-- tests/unit/test_init.py | 22 +++++++++++++++++++++- 5 files changed, 62 insertions(+), 12 deletions(-) diff --git a/src/evalshift_cli/cli/commands/init.py b/src/evalshift_cli/cli/commands/init.py index e15b192..aee824f 100644 --- a/src/evalshift_cli/cli/commands/init.py +++ b/src/evalshift_cli/cli/commands/init.py @@ -172,6 +172,20 @@ # See docs/agents.md. """ +#: The scaffold's ``captures:`` block, between the migration policy and the suites region. +#: Public so a test that needs today's unbounded sync can strip it from the scaffold. +CAPTURES_SCAFFOLD: Final = """\ +# How captures become suites. Each suite stays a capped sample of production +# from the last max_age, recorded on defaults.source_model: `capture sync` +# swaps stale cases for fresh captures as they arrive. Does the SDK ship +# captures to a bucket (EVALSHIFT_CAPTURE_STORE on the agent)? Uncomment +# store: and name the same bucket. +captures: + max_age: 14d + max_cases: 200 + # store: s3:/// +""" + def scaffold_embedding_model(provider: str, env: Mapping[str, str]) -> str: """Pick the embedding model ``init`` writes for ``provider``. @@ -241,7 +255,17 @@ def render_minimal_config( # diff stays confined to the tail and everything a person hand-edits above # it keeps stable line numbers. suites = render_suites_region("suites: {}") - return header + body.rstrip() + "\n\n" + policy.rstrip() + "\n\n" + suites + "\n" + return ( + header + + body.rstrip() + + "\n\n" + + policy.rstrip() + + "\n\n" + + CAPTURES_SCAFFOLD + + "\n" + + suites + + "\n" + ) def init( diff --git a/tests/unit/test_cli_capture.py b/tests/unit/test_cli_capture.py index b99ce44..94fa0e9 100644 --- a/tests/unit/test_cli_capture.py +++ b/tests/unit/test_cli_capture.py @@ -12,7 +12,7 @@ from evalshift_cli.captures.models import CaptureEnvelope, PromotedCase from evalshift_cli.captures.toolset import EMPTY_TOOLSET_FINGERPRINT, fingerprint_tools from evalshift_cli.cli.commands.capture import _build_suite_entries, _declared_tool_properties -from evalshift_cli.cli.commands.init import render_minimal_config +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config from evalshift_cli.cli.main import app from evalshift_cli.config.loader import load_config from evalshift_cli.suite.loader import load_jsonl @@ -21,8 +21,13 @@ def _write_min_config(path: Path) -> None: - """Write a minimal init-style config (with the managed suites markers).""" - path.write_text(render_minimal_config(profile="model-upgrade"), encoding="utf-8") + """Write a minimal init-style config (with the managed suites markers). + + Without the scaffold's capture limits: these tests pin unbounded sync, and their + fixtures' June 2026 captures are older than the scaffold's 14d max_age. + """ + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + path.write_text(text, encoding="utf-8") def _write_empty_capture(base: Path, *, capture_id: str, suite: str) -> Path: diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index 1e95c8d..f31c6f9 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -11,7 +11,7 @@ from evalshift_cli.captures.remote import RemoteStore, RemoteStoreUnavailable from evalshift_cli.captures.store_uri import StoreURI from evalshift_cli.cli.commands import capture as capture_module -from evalshift_cli.cli.commands.init import render_minimal_config +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config from evalshift_cli.cli.main import app from evalshift_cli.suite.loader import load_jsonl from tests.unit.fake_remote_store import FakeRemoteStore, populated_store @@ -24,7 +24,7 @@ def _invoke(args: list[str], base: Path) -> Any: def _write_config(path: Path, *, store: str | None = "s3://acme-evals/support-agent") -> Path: - text = render_minimal_config(profile="model-upgrade") + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") if store is not None: text += f"\ncaptures:\n store: {store}\n" path.write_text(text, encoding="utf-8") @@ -228,10 +228,10 @@ def test_invalid_init_config_with_captures_only_in_comments_stays_silent( tmp_path: Path, remote: FakeRemoteStore, command: str ) -> None: # `evalshift init` output mentions "captures" in comments; that is not a store request. + # Its top-level `captures:` block is stripped so only the comments remain. config = tmp_path / "evalshift.yaml" - config.write_text( - render_minimal_config(profile="model-upgrade") + "\ntypo_key: 1\n", encoding="utf-8" - ) + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + config.write_text(text + "\ntypo_key: 1\n", encoding="utf-8") assert b"captures" in config.read_bytes() result = _invoke([command, "--config", str(config)], tmp_path) assert result.exit_code == 0, result.output diff --git a/tests/unit/test_cli_capture_repair.py b/tests/unit/test_cli_capture_repair.py index 978982a..2d338f4 100644 --- a/tests/unit/test_cli_capture_repair.py +++ b/tests/unit/test_cli_capture_repair.py @@ -18,7 +18,7 @@ from evalshift_cli.captures.remote import RemoteStore from evalshift_cli.captures.store_uri import StoreURI from evalshift_cli.cli.commands import capture as capture_module -from evalshift_cli.cli.commands.init import render_minimal_config +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config from evalshift_cli.cli.main import app from evalshift_cli.suite.models import SuiteExample from tests.unit.fake_remote_store import ( @@ -84,7 +84,8 @@ def _open(parsed: StoreURI) -> RemoteStore: def _config(base: Path) -> Path: path = base / "evalshift.yaml" path.write_text( - render_minimal_config(profile="model-upgrade") + "\ncaptures:\n store: s3://b/p\n", + render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + + "\ncaptures:\n store: s3://b/p\n", encoding="utf-8", ) return path diff --git a/tests/unit/test_init.py b/tests/unit/test_init.py index b40a8ec..15b6b5d 100644 --- a/tests/unit/test_init.py +++ b/tests/unit/test_init.py @@ -10,6 +10,7 @@ from __future__ import annotations import re +from datetime import timedelta from pathlib import Path import pytest @@ -34,9 +35,15 @@ SUITES_MARKER_END, ) from evalshift_cli.cli.commands.doctor import CONFIG_FILENAME -from evalshift_cli.cli.commands.init import _PROVIDER_MODELS, PROVIDERS, render_minimal_config +from evalshift_cli.cli.commands.init import ( + _PROVIDER_MODELS, + CAPTURES_SCAFFOLD, + PROVIDERS, + render_minimal_config, +) from evalshift_cli.cli.main import app from evalshift_cli.config.loader import load_config +from evalshift_cli.config.models import EvalShiftConfig from evalshift_cli.models.registry import PROVIDER_ENV_VARS, resolve_model runner = CliRunner() @@ -622,3 +629,16 @@ def test_plain_init_warns_next_to_a_stale_workflow( assert result.exit_code == 0, result.stdout assert "CI installs evalshift 0.0.1" in result.stdout assert 'evalshift-version: "1.2.3"' in result.stdout + + +def test_scaffold_bounds_capture_suites() -> None: + text = render_minimal_config(profile="model-upgrade") + assert CAPTURES_SCAFFOLD in text + cfg = EvalShiftConfig.model_validate(yaml.safe_load(text)) + assert cfg.captures.max_age == timedelta(days=14) + assert cfg.captures.max_cases == 200 + assert cfg.captures.store is None + # The managed suites region stays last: capture sync rewrites only the tail. + block = re.search(r"(?m)^captures:$", text) + assert block is not None + assert block.start() < text.index(SUITES_MARKER_BEGIN) From e24febe87f0ed5473760d30c818457fafce3bde0 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:12:28 +0200 Subject: [PATCH 05/26] feat(models): match recorded model ids against the source model Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/models/matching.py | 52 ++++++++++++++++++++++++++ tests/unit/test_model_matching.py | 56 ++++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 src/evalshift_cli/models/matching.py create mode 100644 tests/unit/test_model_matching.py diff --git a/src/evalshift_cli/models/matching.py b/src/evalshift_cli/models/matching.py new file mode 100644 index 0000000..a88d584 --- /dev/null +++ b/src/evalshift_cli/models/matching.py @@ -0,0 +1,52 @@ +"""Whether two model ids name the same model, for capture-suite staleness. + +A capture records whatever id the provider echoed back -- often a dated snapshot +(``claude-sonnet-5-20260301``) or a provider-prefixed id -- while ``defaults.source_model`` +is usually the bare alias. Matching errs toward "same": a false match keeps a stale case a +little longer, a false mismatch would retire good ones. +""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable + +from evalshift_cli.models.registry import resolve_model + +_SNAPSHOT_SUFFIX = re.compile(r"-(?:\d{8}|\d{4}-\d{2}-\d{2})$") + + +def bare_model_name(model_id: str) -> str: + """``model_id`` lower-cased, without its provider prefix or a trailing snapshot date.""" + name = model_id.strip().lower().rsplit("/", 1)[-1] + return _SNAPSHOT_SUFFIX.sub("", name) + + +def same_model(a: str, b: str) -> bool: + """True when ``a`` and ``b`` share a bare name or resolve to one registry id.""" + if not a.strip() or not b.strip(): + return False + if bare_model_name(a) == bare_model_name(b): + return True + return resolve_model(a).id == resolve_model(b).id + + +def model_filter_applies(recorded: Iterable[str], source_model: str | None) -> bool: + """Whether filtering one suite by ``source_model`` is safe (the spec §3 guardrail). + + False without a source model, when nothing recorded a model, and when not one recorded + model matches it -- a wrong or unrecognised ``defaults.source_model``, or a migration whose + first new-model capture has not arrived yet, must never empty or freeze a suite. + """ + if not source_model: + return False + return any(same_model(model, source_model) for model in recorded if model) + + +def describe_models(recorded: Iterable[str], *, limit: int = 3) -> str: + """``a ×3, c ×2, b ×1, +1 more``: the most-recorded ids first, at most ``limit`` of them.""" + counts = Counter(model for model in recorded if model) + shown = ", ".join(f"{model} ×{n}" for model, n in counts.most_common(limit)) + rest = len(counts) - limit + return f"{shown}, +{rest} more" if rest > 0 else shown diff --git a/tests/unit/test_model_matching.py b/tests/unit/test_model_matching.py new file mode 100644 index 0000000..5d75aaf --- /dev/null +++ b/tests/unit/test_model_matching.py @@ -0,0 +1,56 @@ +"""Matching a capture's recorded model id against ``defaults.source_model``.""" + +from __future__ import annotations + +import pytest + +from evalshift_cli.models.matching import describe_models, model_filter_applies, same_model + + +@pytest.mark.parametrize( + ("a", "b"), + [ + ("claude-sonnet-5-20260301", "claude-sonnet-5"), + ("anthropic/claude-sonnet-5", "claude-sonnet-5"), + ("google/gemini-3.8-flash", "gemini-3.8-flash"), + ("gpt-5-2026-08-07", "openai/gpt-5"), + ("Claude-Sonnet-5", "claude-sonnet-5"), + ], +) +def test_same_model_matches_prefix_case_and_snapshot_variants(a: str, b: str) -> None: + assert same_model(a, b) + assert same_model(b, a) + + +@pytest.mark.parametrize( + ("a", "b"), + [ + ("claude-sonnet-5", "claude-sonnet-5-5"), + ("gpt-5", "gpt-5-mini"), + ("", "claude-sonnet-5"), + ("m", "claude-sonnet-5"), + ], +) +def test_same_model_keeps_distinct_models_apart(a: str, b: str) -> None: + assert not same_model(a, b) + + +def test_filter_applies_when_any_recorded_model_matches() -> None: + assert model_filter_applies(["claude-sonnet-5-20260301", "gpt-5"], "claude-sonnet-5") + + +def test_filter_is_off_without_a_source_model() -> None: + assert not model_filter_applies(["claude-sonnet-5"], None) + + +def test_filter_is_off_when_nothing_recorded_a_model() -> None: + assert not model_filter_applies(["", ""], "claude-sonnet-5") + + +def test_guardrail_filter_is_off_when_no_recorded_model_matches() -> None: + assert not model_filter_applies(["claude-sonnet-5"] * 3, "claude-sonnet-5-5") + + +def test_describe_models_counts_the_most_recorded_first() -> None: + recorded = ["a"] * 3 + ["b"] + ["c"] * 2 + ["d"] + [""] + assert describe_models(recorded) == "a ×3, c ×2, b ×1, +1 more" From a04f41d9b966f9a800ecf7824e5db078fd6f590e Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:18:03 +0200 Subject: [PATCH 06/26] fix(models): match model ids on bare names only Co-Authored-By: Claude Opus 5.5 --- ...2026-10-16-bounded-capture-suites-design.md | 10 +++++----- src/evalshift_cli/models/matching.py | 18 ++++++++---------- tests/unit/test_model_matching.py | 2 ++ 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md index 9008c57..ecf13c0 100644 --- a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md +++ b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md @@ -93,11 +93,11 @@ One clock instant `now` per sync run (a module-level `_now()` so tests can pin i `same_model(source_model, defaults.source_model)` is false. Only when bounded suites are on (§1) and `defaults.source_model` is set. -`same_model(a, b)` — new `models/matching.py`, no config import. True when either: - -1. the **bare names** are equal, where bare = lowercase, drop everything up to the last `/`, then - strip one trailing snapshot date `-YYYYMMDD` or `-YYYY-MM-DD`; or -2. `resolve_model(a).id == resolve_model(b).id` (registry aliases). +`same_model(a, b)` — new `models/matching.py`, no config import. True when the **bare names** are +equal, where bare = lowercase, drop everything up to the last `/`, then strip one trailing +snapshot date `-YYYYMMDD` or `-YYYY-MM-DD`. Matching is on bare names only — no registry lookup — +because the registry's forward aliases (`gpt-5` → `openai/gpt-4o`) would make distinct models +match. | a | b | match | via | | --- | --- | --- | --- | diff --git a/src/evalshift_cli/models/matching.py b/src/evalshift_cli/models/matching.py index a88d584..f438f79 100644 --- a/src/evalshift_cli/models/matching.py +++ b/src/evalshift_cli/models/matching.py @@ -1,9 +1,11 @@ """Whether two model ids name the same model, for capture-suite staleness. -A capture records whatever id the provider echoed back -- often a dated snapshot -(``claude-sonnet-5-20260301``) or a provider-prefixed id -- while ``defaults.source_model`` -is usually the bare alias. Matching errs toward "same": a false match keeps a stale case a -little longer, a false mismatch would retire good ones. +A capture records the ``model`` the application passed in its request -- which may be a dated +snapshot (``claude-sonnet-5-20260301``) or a provider-prefixed id -- while +``defaults.source_model`` is usually the bare alias. Matching compares bare names only: the +model registry's forward aliases (``gpt-5`` -> ``openai/gpt-4o``) would conflate distinct +models, and a source model spelled as a registry-only alias trips the guardrail warning rather +than matching silently. """ from __future__ import annotations @@ -12,8 +14,6 @@ from collections import Counter from collections.abc import Iterable -from evalshift_cli.models.registry import resolve_model - _SNAPSHOT_SUFFIX = re.compile(r"-(?:\d{8}|\d{4}-\d{2}-\d{2})$") @@ -24,12 +24,10 @@ def bare_model_name(model_id: str) -> str: def same_model(a: str, b: str) -> bool: - """True when ``a`` and ``b`` share a bare name or resolve to one registry id.""" + """True when ``a`` and ``b`` are both non-blank and share a bare name.""" if not a.strip() or not b.strip(): return False - if bare_model_name(a) == bare_model_name(b): - return True - return resolve_model(a).id == resolve_model(b).id + return bare_model_name(a) == bare_model_name(b) def model_filter_applies(recorded: Iterable[str], source_model: str | None) -> bool: diff --git a/tests/unit/test_model_matching.py b/tests/unit/test_model_matching.py index 5d75aaf..289a519 100644 --- a/tests/unit/test_model_matching.py +++ b/tests/unit/test_model_matching.py @@ -27,6 +27,8 @@ def test_same_model_matches_prefix_case_and_snapshot_variants(a: str, b: str) -> [ ("claude-sonnet-5", "claude-sonnet-5-5"), ("gpt-5", "gpt-5-mini"), + ("gpt-4o", "gpt-5"), + ("gpt-4o-mini", "gpt-5-mini"), ("", "claude-sonnet-5"), ("m", "claude-sonnet-5"), ], From 2331bda751d76000fc5ed78548a086ac5fe26364 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:24:25 +0200 Subject: [PATCH 07/26] feat(captures): promoted cases record their capture time and model Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/models.py | 7 +++ src/evalshift_cli/captures/promote.py | 25 +++++++++ src/evalshift_cli/cli/commands/capture.py | 65 +++++++++++++++++++++++ tests/unit/fake_remote_store.py | 3 +- tests/unit/test_captures_models.py | 8 +++ tests/unit/test_captures_promote.py | 39 +++++++++++++- tests/unit/test_cli_capture.py | 62 +++++++++++++++++++-- tests/unit/test_cli_capture_repair.py | 4 ++ 8 files changed, 208 insertions(+), 5 deletions(-) diff --git a/src/evalshift_cli/captures/models.py b/src/evalshift_cli/captures/models.py index 5edc6b3..6252e35 100644 --- a/src/evalshift_cli/captures/models.py +++ b/src/evalshift_cli/captures/models.py @@ -86,6 +86,13 @@ class PromotedCase(BaseModel): # reproduce it. Defaults keep pre-existing case files parsing. cost_usd: float = Field(default=0.0, ge=0.0) cost_source: Literal["recorded", "estimated"] | None = None + # When the source capture was recorded (its ``created_at``, verbatim) and the model its + # first ``model_call`` ran on -- the call a replay re-sends. With captures.max_age / + # captures.max_cases set, ``capture sync`` uses both to tell a stale case from a fresh + # one. Empty on case files written before they existed; sync backfills them while the + # capture is still local. + captured_at: str = "" + source_model: str = "" # Multi-turn provenance — optional so existing promoted-case JSON files # (written before conversation tracking) still parse. conversation_id: str | None = None diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index 44241bc..e1125fb 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -1540,6 +1540,29 @@ def write_promoted_case( return path +def capture_provenance(envelope: CaptureEnvelope) -> tuple[str, str]: + """``(captured_at, source_model)`` for a case promoted from ``envelope``. + + ``source_model`` is the first ``model_call``'s ``model_id`` -- the call whose input a replay + re-sends -- or ``""`` when the capture recorded none. + """ + first = next((e for e in envelope.trace.events if isinstance(e, ModelCallEvent)), None) + return envelope.created_at, first.model_id if first is not None else "" + + +def retire_promoted_case(suite: str, name: str, *, base: Path | None = None) -> bool: + """Delete ``/suites//.json``; ``False`` when it was already gone. + + The caller rebuilds ``golden.jsonl`` afterwards (:func:`rebuild_golden_jsonl`). + """ + path = promoted_suite_dir(suite, base=base) / f"{_safe_segment(name)}.json" + try: + path.unlink() + except FileNotFoundError: + return False + return True + + def example_content_key(example: SuiteExample) -> str: """A stable key over the *replayed content* of ``example`` (inputs + history). @@ -1598,11 +1621,13 @@ def rebuild_golden_jsonl(suite_dir: Path) -> Path: "RecoveredInput", "build_conversation_examples", "build_example_from_capture", + "capture_provenance", "duplicate_turn_warnings", "example_content_key", "iter_promoted_cases", "promoted_suite_dir", "rebuild_golden_jsonl", "recover_from_messages", + "retire_promoted_case", "write_promoted_case", ] diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 3abb8d0..70d0aa8 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -33,6 +33,7 @@ PromoteOptions, build_conversation_examples, build_example_from_capture, + capture_provenance, duplicate_turn_warnings, example_content_key, iter_promoted_cases, @@ -229,6 +230,56 @@ def _promoted_content_owners( return owners +def _load_promoted(suite: str, *, base: Path | None) -> tuple[list[PromotedCase], str | None]: + """The suite's promoted cases, plus the error's type name when its files could not be read. + + A suite with no directory yet has no cases. Only the exception's type name is kept: its + text can carry raw bytes of a case file. + """ + suite_dir = promoted_suite_dir(suite, base=base) + if not suite_dir.is_dir(): + return [], None + try: + return iter_promoted_cases(suite_dir), None + except (ValueError, OSError) as exc: # pydantic's ValidationError is a ValueError + return [], type(exc).__name__ + + +def _backfill_provenance( + cases: list[PromotedCase], + envelopes: Mapping[str, CaptureEnvelope], + *, + base: Path | None, +) -> tuple[list[PromotedCase], int]: + """Fill ``captured_at`` / ``source_model`` on cases written before the fields existed. + + Only an empty field is filled, only from the case's own capture, and only while that + capture is local; nothing else on the case changes (``promoted_at`` included). Returns + the cases as they now stand and how many files were rewritten. + """ + out: list[PromotedCase] = [] + rewritten = 0 + for case in cases: + envelope = envelopes.get(case.from_capture) + if envelope is None or (case.captured_at and case.source_model): + out.append(case) + continue + captured_at, source_model = capture_provenance(envelope) + filled = case.model_copy( + update={ + "captured_at": case.captured_at or captured_at, + "source_model": case.source_model or source_model, + } + ) + if filled == case: + out.append(case) + continue + write_promoted_case(filled, base=base, force=True) + out.append(filled) + rewritten += 1 + return out, rewritten + + def _repair_selection(*, base: Path | None, suite: str | None, console: Console) -> RepairSelection: """The promoted cases ``capture sync`` treats as repair candidates (``captures/repair.py``). @@ -721,6 +772,7 @@ def capture_promote( "together.", ) + captured_at, source_model = capture_provenance(envelope) case = PromotedCase( name=opts.name or envelope.capture_id, suite=envelope.suite, @@ -733,6 +785,8 @@ def capture_promote( cost_source=built.cost_source, conversation_id=built.example.conversation_id, turn_index=built.example.turn_index, + captured_at=captured_at, + source_model=source_model, example=built.example, ) @@ -1146,6 +1200,7 @@ def capture_sync( skipped_invalid = 0 repaired = 0 left_for_repair = 0 + backfilled = 0 backed_by = selection.cases_by_capture() suites_seen: set[str] = set() # (suite, content key) -> the name of the case that owns that content, @@ -1175,6 +1230,11 @@ def capture_sync( [r.envelope for r in records_by_suite[suite_name]], ): console.print(f"[yellow]⚠[/yellow] {suite_name}: {warning}") + envelopes = {r.envelope.capture_id: r.envelope for r in records_by_suite[suite_name]} + kept_cases, unreadable = _load_promoted(suite_name, base=base) + if unreadable is None: + kept_cases, rewritten = _backfill_provenance(kept_cases, envelopes, base=base) + backfilled += rewritten built_pairs = build_conversation_examples(records_by_suite[suite_name], opts, base=base) local_ids = {r.envelope.capture_id for r in records_by_suite[suite_name]} # Re-exercising an agent on the same data records duplicate captures. @@ -1295,6 +1355,7 @@ def capture_sync( seen_examples.setdefault(content_key, envelope.capture_id) for warning in built.warnings: console.print(f"[yellow]⚠[/yellow] {envelope.capture_id}: {warning}") + captured_at, source_model = capture_provenance(envelope) case = PromotedCase( name=envelope.capture_id, suite=envelope.suite, @@ -1307,6 +1368,8 @@ def capture_sync( cost_source=built.cost_source, conversation_id=built.example.conversation_id, turn_index=built.example.turn_index, + captured_at=captured_at, + source_model=source_model, example=built.example, ) try: @@ -1340,6 +1403,8 @@ def capture_sync( summary = f"[green]✓[/green] promoted {promoted} capture(s) into {len(suite_paths)} suite(s)" if wired_generation: summary += f", wired generation config for {wired_generation} case(s)" + if backfilled: + summary += f", backfilled capture time and model on {backfilled} case(s)" if skipped_existing: summary += f", skipped {skipped_existing} already-promoted (use --force to overwrite)" if skipped_empty: diff --git a/tests/unit/fake_remote_store.py b/tests/unit/fake_remote_store.py index 7564445..ccaa7d5 100644 --- a/tests/unit/fake_remote_store.py +++ b/tests/unit/fake_remote_store.py @@ -23,6 +23,7 @@ def capture_payload( created_at: str = "2026-06-16T12:00:00+00:00", conversation_id: str | None = None, turn_index: int | None = None, + model_id: str = "m", ) -> bytes: """A minimal capture the CLI reader accepts: one model call, one final output.""" envelope = { @@ -45,7 +46,7 @@ def capture_payload( "sequence_index": 0, "timestamp": created_at, "metadata": {}, - "model_id": "m", + "model_id": model_id, "input": model_input, "output": "out", "toolset_ref": TOOLSET_REF, diff --git a/tests/unit/test_captures_models.py b/tests/unit/test_captures_models.py index 5d9a0c2..cd4a341 100644 --- a/tests/unit/test_captures_models.py +++ b/tests/unit/test_captures_models.py @@ -112,3 +112,11 @@ def test_promotion_source_round_trips(self) -> None: def test_promotion_source_rejects_an_unknown_yardstick(self) -> None: with pytest.raises(ValidationError, match="promotion_source"): PromotedCase.model_validate(self._case_payload(promotion_source="guessed")) + + +def test_a_case_written_before_capture_provenance_still_parses() -> None: + case = PromotedCase.model_validate( + {"name": "c", "suite": "s", "from_capture": "cap_1", "example": {"id": "c", "tools": []}} + ) + assert case.captured_at == "" + assert case.source_model == "" diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index bff8eb1..083dfda 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -25,15 +25,17 @@ _tool_rounds, build_conversation_examples, build_example_from_capture, + capture_provenance, duplicate_turn_warnings, example_content_key, rebuild_golden_jsonl, + retire_promoted_case, write_promoted_case, ) from evalshift_cli.captures.reader import CaptureRecord from evalshift_cli.captures.toolset import EMPTY_TOOLSET_FINGERPRINT from evalshift_cli.suite.loader import load_jsonl -from evalshift_cli.suite.models import HistoryToolCall +from evalshift_cli.suite.models import HistoryToolCall, SuiteExample FIXTURES = Path(__file__).parent / "fixtures" / "captures" MULTI_ROUND_CAPTURE = FIXTURES / "multi_round_tools.json" @@ -2999,3 +3001,38 @@ def test_an_invalid_case_in_a_conversation_does_not_seed_later_turns() -> None: assert by_id["cap_abc"].blocked_reason == "invalid_case" assert by_id["cap_t1"].blocked is None assert by_id["cap_t1"].example.history == [] + + +class TestCaptureProvenance: + def test_reads_created_at_and_the_first_model_calls_model(self) -> None: + envelope = _envelope( + events=[ + _model_call(0, model_input="hi", model_id="claude-sonnet-5-20260301"), + _model_call(1, model_input="again", model_id="claude-haiku-5"), + _final("done", 2), + ], + created_at="2026-10-01T12:00:00+00:00", + ) + assert capture_provenance(envelope) == ( + "2026-10-01T12:00:00+00:00", + "claude-sonnet-5-20260301", + ) + + def test_a_capture_without_a_model_call_has_no_model(self) -> None: + assert capture_provenance(_envelope(events=[_final("done", 0)]))[1] == "" + + +class TestRetirePromotedCase: + def test_deletes_the_case_file(self, tmp_path: Path) -> None: + case = PromotedCase( + name="cap_1", + suite="alpha", + from_capture="cap_1", + example=SuiteExample(id="cap_1", tools=[]), + ) + path = write_promoted_case(case, base=tmp_path) + assert retire_promoted_case("alpha", "cap_1", base=tmp_path) is True + assert not path.exists() + + def test_an_absent_case_is_not_an_error(self, tmp_path: Path) -> None: + assert retire_promoted_case("alpha", "cap_nope", base=tmp_path) is False diff --git a/tests/unit/test_cli_capture.py b/tests/unit/test_cli_capture.py index 94fa0e9..04bec11 100644 --- a/tests/unit/test_cli_capture.py +++ b/tests/unit/test_cli_capture.py @@ -55,7 +55,7 @@ def _write_empty_capture(base: Path, *, capture_id: str, suite: str) -> Path: def _events( - *, tools: list[str], model_input: Any = "where is order 12345?" + *, tools: list[str], model_input: Any = "where is order 12345?", model_id: str = "m" ) -> list[dict[str, Any]]: events: list[dict[str, Any]] = [ { @@ -63,7 +63,7 @@ def _events( "sequence_index": 0, "timestamp": "2026-06-16T12:00:00+00:00", "metadata": {}, - "model_id": "m", + "model_id": model_id, "input": model_input, "output": "out", "toolset_ref": "sha256:" + "ab" * 32, @@ -128,6 +128,7 @@ def _write_capture( conversation_id: str | None = None, turn_index: int | None = None, input_hash: str | None = None, + model_id: str = "m", ) -> Path: _write_toolset_sidecar(base) payload = { @@ -147,7 +148,9 @@ def _write_capture( "events": events if events is not None else _events( - tools=tools if tools is not None else ["search_orders"], model_input=model_input + tools=tools if tools is not None else ["search_orders"], + model_input=model_input, + model_id=model_id, ), }, "conversation_id": conversation_id, @@ -2291,3 +2294,56 @@ def test_promote_invalid_case_refusal_does_not_suggest_allow_errored(tmp_path: P assert "could not be turned into a valid case" in out assert "--allow-errored" not in out assert not (tmp_path / "suites" / "alpha" / "cap_bad.json").exists() + + +def test_sync_records_when_and_on_what_each_capture_ran(tmp_path: Path) -> None: + _write_capture( + tmp_path, + capture_id="cap_1", + suite="alpha", + created_at="2026-10-01T09:30:00+00:00", + model_id="claude-sonnet-5-20260301", + ) + config = tmp_path / "evalshift.yaml" + _write_min_config(config) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + case = PromotedCase.model_validate_json( + (tmp_path / "suites" / "alpha" / "cap_1.json").read_text(encoding="utf-8") + ) + assert case.captured_at == "2026-10-01T09:30:00+00:00" + assert case.source_model == "claude-sonnet-5-20260301" + + +def test_sync_backfills_a_case_written_before_the_fields_existed(tmp_path: Path) -> None: + _write_capture(tmp_path, capture_id="cap_1", suite="alpha", model_id="claude-sonnet-5") + config = tmp_path / "evalshift.yaml" + _write_min_config(config) + assert _invoke(["sync", "--config", str(config)], tmp_path).exit_code == 0 + case_path = tmp_path / "suites" / "alpha" / "cap_1.json" + legacy = json.loads(case_path.read_text(encoding="utf-8")) + del legacy["captured_at"], legacy["source_model"] + case_path.write_text(json.dumps(legacy), encoding="utf-8") + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + case = PromotedCase.model_validate_json(case_path.read_text(encoding="utf-8")) + assert case.captured_at == "2026-06-16T12:00:00+00:00" + assert case.source_model == "claude-sonnet-5" + assert case.promoted_at == legacy["promoted_at"] + assert "backfilled capture time and model on 1 case(s)" in " ".join(result.stdout.split()) + + +def test_promote_records_when_and_on_what_the_capture_ran(tmp_path: Path) -> None: + _write_capture(tmp_path, capture_id="cap_1", model_id="gpt-5-2026-08-07") + + assert _invoke(["promote", "cap_1", "--as", "case1"], tmp_path).exit_code == 0 + + case = PromotedCase.model_validate_json( + (tmp_path / "suites" / "support_agent" / "case1.json").read_text(encoding="utf-8") + ) + assert case.source_model == "gpt-5-2026-08-07" + assert case.captured_at == "2026-06-16T12:00:00+00:00" diff --git a/tests/unit/test_cli_capture_repair.py b/tests/unit/test_cli_capture_repair.py index 2d338f4..0b2a54f 100644 --- a/tests/unit/test_cli_capture_repair.py +++ b/tests/unit/test_cli_capture_repair.py @@ -156,12 +156,16 @@ def _write_case( "conversation_id": conversation_id, } ) + # Provenance matches what ``capture_payload`` records, so sync's backfill leaves the + # file alone and the byte-for-byte "untouched" checks below measure repair only. case = PromotedCase( name=case_name, suite=suite, from_capture=capture_id, example=example, conversation_id=conversation_id, + captured_at="2026-06-16T12:00:00+00:00", + source_model="m", ) path = base / "suites" / suite / f"{case_name}.json" path.parent.mkdir(parents=True, exist_ok=True) From efae223654aa61d022fe23587ded4f29b8a70fec Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:30:04 +0200 Subject: [PATCH 08/26] fix(capture): sync --repair keeps backfilled capture provenance Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 33 ++++++++++++++++------- tests/unit/test_cli_capture.py | 7 +++++ tests/unit/test_cli_capture_repair.py | 24 +++++++++++++++++ 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 70d0aa8..855a33b 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -245,6 +245,23 @@ def _load_promoted(suite: str, *, base: Path | None) -> tuple[list[PromotedCase] return [], type(exc).__name__ +def _with_provenance(case: PromotedCase, envelope: CaptureEnvelope) -> PromotedCase: + """``case`` with an empty ``captured_at`` / ``source_model`` filled from ``envelope``. + + ``envelope`` is the case's own capture. A field already set is kept, and nothing else on + the case changes; ``case`` itself comes back when there is nothing to fill. + """ + if case.captured_at and case.source_model: + return case + captured_at, source_model = capture_provenance(envelope) + return case.model_copy( + update={ + "captured_at": case.captured_at or captured_at, + "source_model": case.source_model or source_model, + } + ) + + def _backfill_provenance( cases: list[PromotedCase], envelopes: Mapping[str, CaptureEnvelope], @@ -261,16 +278,10 @@ def _backfill_provenance( rewritten = 0 for case in cases: envelope = envelopes.get(case.from_capture) - if envelope is None or (case.captured_at and case.source_model): + if envelope is None: out.append(case) continue - captured_at, source_model = capture_provenance(envelope) - filled = case.model_copy( - update={ - "captured_at": case.captured_at or captured_at, - "source_model": case.source_model or source_model, - } - ) + filled = _with_provenance(case, envelope) if filled == case: out.append(case) continue @@ -1318,7 +1329,11 @@ def capture_sync( "store, so its history can't be rebuilt", ) continue - write_promoted_case(case, base=base, force=True) + # ``backed`` was loaded before this suite's backfill ran, so the + # repaired case would otherwise drop the provenance just written. + write_promoted_case( + _with_provenance(case, envelope), base=base, force=True + ) repaired_here.append(case.name) continue if built.blocked is not None: diff --git a/tests/unit/test_cli_capture.py b/tests/unit/test_cli_capture.py index 04bec11..b152313 100644 --- a/tests/unit/test_cli_capture.py +++ b/tests/unit/test_cli_capture.py @@ -2335,6 +2335,13 @@ def test_sync_backfills_a_case_written_before_the_fields_existed(tmp_path: Path) assert case.source_model == "claude-sonnet-5" assert case.promoted_at == legacy["promoted_at"] assert "backfilled capture time and model on 1 case(s)" in " ".join(result.stdout.split()) + after_backfill = case_path.read_bytes() + + again = _invoke(["sync", "--config", str(config)], tmp_path) + + assert again.exit_code == 0, again.stdout + assert "backfilled" not in again.stdout + assert case_path.read_bytes() == after_backfill def test_promote_records_when_and_on_what_the_capture_ran(tmp_path: Path) -> None: diff --git a/tests/unit/test_cli_capture_repair.py b/tests/unit/test_cli_capture_repair.py index 0b2a54f..769b340 100644 --- a/tests/unit/test_cli_capture_repair.py +++ b/tests/unit/test_cli_capture_repair.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import shutil from datetime import UTC, datetime from pathlib import Path @@ -325,6 +326,29 @@ def test_repair_twice_is_a_no_op(tmp_path: Path, store: FakeRemoteStore) -> None assert case.read_bytes() == after_first +def test_repair_keeps_the_capture_provenance_it_backfills( + tmp_path: Path, store: FakeRemoteStore +) -> None: + """A 1.3.0-era case has no provenance; repairing it must not drop what sync backfilled.""" + case = _write_case(tmp_path, "cap_1") + legacy = json.loads(case.read_text(encoding="utf-8")) + del legacy["captured_at"], legacy["source_model"] + case.write_text(json.dumps(legacy), encoding="utf-8") + _put(store, "cap_1") + + first = _sync(tmp_path, "--repair") + + assert first.exit_code == 0, first.output + assert "repaired 1 case(s)" in _out(first) + assert _stored(case).captured_at == "2026-06-16T12:00:00+00:00" + assert _stored(case).source_model == "m" + after_first = case.read_bytes() + second = _sync(tmp_path) + assert second.exit_code == 0, second.output + assert "backfilled" not in _out(second) + assert case.read_bytes() == after_first + + def test_repair_leaves_a_case_whose_rebuild_is_blocked( tmp_path: Path, store: FakeRemoteStore ) -> None: From e7dc2cbe00973d9f08707c320f5c83c66944069d Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:34:19 +0200 Subject: [PATCH 09/26] feat(captures): staleness rules for promoted cases Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/staleness.py | 82 +++++++++++++++++++ tests/unit/test_captures_staleness.py | 104 ++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 src/evalshift_cli/captures/staleness.py create mode 100644 tests/unit/test_captures_staleness.py diff --git a/src/evalshift_cli/captures/staleness.py b/src/evalshift_cli/captures/staleness.py new file mode 100644 index 0000000..8f065fb --- /dev/null +++ b/src/evalshift_cli/captures/staleness.py @@ -0,0 +1,82 @@ +"""What makes a promoted case stale -- shared by ``capture sync`` and the freshness checks. + +See ``docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md`` §3. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from typing import Literal + +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.models.matching import same_model +from evalshift_cli.suite.models import SuiteExample + +StaleReason = Literal["age", "model"] +"""Why a case is stale: older than ``captures.max_age``, or recorded on another model.""" + +#: The route of a case that expects no tool call. +NO_TOOLS_ROUTE = "∅" + + +def parse_instant(text: str) -> datetime | None: + """An ISO 8601 timestamp as an aware UTC datetime; naive means UTC; ``None`` if unparseable.""" + cleaned = text.strip() + if not cleaned: + return None + try: + parsed = datetime.fromisoformat(cleaned) + except ValueError: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def case_instant(case: PromotedCase) -> datetime | None: + """When ``case``'s capture was recorded. + + ``captured_at``, else ``promoted_at`` -- an upper bound on the capture time, so a legacy + case looks younger than it is and is retired later, never earlier. + """ + return parse_instant(case.captured_at) or parse_instant(case.promoted_at) + + +def is_reviewed(example: SuiteExample) -> bool: + """Whether a human marked any of ``example``'s tool expectations ``provenance: reviewed``.""" + calls = list(example.expected_tools or []) + for round_calls in example.expected_tool_rounds or []: + calls.extend(round_calls) + return any(call.provenance == "reviewed" for call in calls) + + +def examples_route(examples: Iterable[SuiteExample]) -> str: + """The sorted, comma-joined round-1 tool names across ``examples``; ``∅`` for none.""" + names = sorted({call.tool_name for e in examples for call in e.expected_tools or []}) + return ",".join(names) if names else NO_TOOLS_ROUTE + + +@dataclass(frozen=True, slots=True) +class StalenessRules: + """The limits one sync (or one freshness check) measures cases against. + + Attributes: + now: The single instant ages are measured from. + max_age: ``captures.max_age``; ``None`` turns age staleness off. + source_model: The model a case must have been recorded on; ``None`` turns model + staleness off (no ``defaults.source_model``, or the guardrail tripped). + """ + + now: datetime + max_age: timedelta | None = None + source_model: str | None = None + + def stale_reason(self, instant: datetime | None, model: str) -> StaleReason | None: + """``model`` before ``age``; ``None`` when fresh or when the fact is unknown.""" + if self.source_model and model and not same_model(model, self.source_model): + return "model" + if self.max_age is not None and instant is not None and instant < self.now - self.max_age: + return "age" + return None diff --git a/tests/unit/test_captures_staleness.py b/tests/unit/test_captures_staleness.py new file mode 100644 index 0000000..51540fb --- /dev/null +++ b/tests/unit/test_captures_staleness.py @@ -0,0 +1,104 @@ +"""What makes a promoted case stale (spec §3).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.staleness import ( + NO_TOOLS_ROUTE, + StalenessRules, + case_instant, + examples_route, + is_reviewed, + parse_instant, +) +from evalshift_cli.suite.models import ExpectedToolCall, SuiteExample + +NOW = datetime(2026, 10, 16, 12, tzinfo=UTC) +#: A non-empty toolset (expected_tools may not sit beside an empty one). +_REF = "sha256:" + "cd" * 32 + + +def _example(*tools: str, reviewed: bool = False) -> SuiteExample: + calls = [ + ExpectedToolCall(tool_name=t, provenance="reviewed" if reviewed else "captured") + for t in tools + ] + return SuiteExample(id="c", toolset_ref=_REF, expected_tools=calls or None) + + +def _case(*, captured_at: str = "", promoted_at: str = "") -> PromotedCase: + return PromotedCase( + name="c", + suite="s", + from_capture="cap", + captured_at=captured_at, + promoted_at=promoted_at, + example=_example(), + ) + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("2026-10-01T12:00:00+00:00", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ("2026-10-01T12:00:00Z", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ("2026-10-01T14:00:00+02:00", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ("2026-10-01T12:00:00", datetime(2026, 10, 1, 12, tzinfo=UTC)), + ], +) +def test_parse_instant_normalises_to_utc(text: str, expected: datetime) -> None: + assert parse_instant(text) == expected + + +@pytest.mark.parametrize("text", ["", " ", "yesterday", "2026-13-01"]) +def test_parse_instant_is_none_for_anything_unparseable(text: str) -> None: + assert parse_instant(text) is None + + +def test_case_instant_prefers_the_capture_time() -> None: + case = _case(captured_at="2026-10-01T00:00:00+00:00", promoted_at="2026-10-05T00:00:00+00:00") + assert case_instant(case) == datetime(2026, 10, 1, tzinfo=UTC) + + +def test_case_instant_falls_back_to_the_promotion_time() -> None: + case = _case(promoted_at="2026-10-05T00:00:00+00:00") + assert case_instant(case) == datetime(2026, 10, 5, tzinfo=UTC) + + +def test_case_instant_is_none_when_neither_parses() -> None: + assert case_instant(_case()) is None + + +def test_is_reviewed_reads_any_expectation() -> None: + assert is_reviewed(_example("a", reviewed=True)) + assert not is_reviewed(_example("a")) + assert not is_reviewed(_example()) + + +def test_route_is_the_sorted_union_of_expected_tool_names() -> None: + assert examples_route([_example("b", "a")]) == "a,b" + assert examples_route([_example("a"), _example("c")]) == "a,c" + assert examples_route([_example()]) == NO_TOOLS_ROUTE + + +def test_an_old_case_is_age_stale() -> None: + rules = StalenessRules(now=NOW, max_age=timedelta(days=14)) + assert rules.stale_reason(NOW - timedelta(days=15), "m") == "age" + assert rules.stale_reason(NOW - timedelta(days=13), "m") is None + assert rules.stale_reason(None, "m") is None + + +def test_another_model_is_model_stale_before_age() -> None: + rules = StalenessRules(now=NOW, max_age=timedelta(days=14), source_model="claude-sonnet-5-5") + old = NOW - timedelta(days=30) + assert rules.stale_reason(old, "claude-sonnet-5-20260301") == "model" + assert rules.stale_reason(old, "claude-sonnet-5-5") == "age" + assert rules.stale_reason(NOW, "") is None + + +def test_no_limits_means_nothing_is_stale() -> None: + assert StalenessRules(now=NOW).stale_reason(NOW - timedelta(days=999), "x") is None From f8c2f8d08e393f1370561b1d3d72d3e33e56975d Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:40:02 +0200 Subject: [PATCH 10/26] feat(captures): plan_suite picks and retires cases for a bounded suite Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/selection.py | 166 ++++++++++++++++++++++++ tests/unit/test_captures_selection.py | 141 ++++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 src/evalshift_cli/captures/selection.py create mode 100644 tests/unit/test_captures_selection.py diff --git a/src/evalshift_cli/captures/selection.py b/src/evalshift_cli/captures/selection.py new file mode 100644 index 0000000..8e483b1 --- /dev/null +++ b/src/evalshift_cli/captures/selection.py @@ -0,0 +1,166 @@ +"""Which captures enter a bounded suite and which promoted cases leave it. + +Pure: no I/O, no clock. ``capture sync`` builds the inputs from disk and applies the plan. +See ``docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md`` §5. +""" + +from __future__ import annotations + +import hashlib +from collections import Counter +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from evalshift_cli.captures.staleness import StaleReason + +RetireReason = Literal["age", "model", "over_cap"] +"""Why a case left: too old, recorded on another model, or trimmed to ``max_cases``.""" + + +@dataclass(frozen=True, slots=True) +class KeptCase: + """A promoted case already in the suite, as the planner sees it. + + Attributes: + name: The case name (its file stem) -- what a retirement deletes. + instant: When its capture was recorded (``staleness.case_instant``), if known. + stale: Why it is stale, or ``None`` when fresh. + reviewed: A human marked one of its expectations ``reviewed``: never retired. + route: Its tool route (``staleness.examples_route``). + content_key: ``promote.example_content_key`` of its example. + """ + + name: str + instant: datetime | None + stale: StaleReason | None + reviewed: bool + route: str + content_key: str + + +@dataclass(frozen=True, slots=True) +class CandidateGroup: + """Eligible captures that enter a suite together: one conversation, or one lone capture. + + Attributes: + key: Stable identity used for the deterministic order. + capture_ids: Its turns' case names (the capture ids), in turn order. + route: The union route of its turns. + content_keys: Each turn's ``example_content_key``. + """ + + key: str + capture_ids: tuple[str, ...] + route: str + content_keys: frozenset[str] + + @property + def size(self) -> int: + """How many cases the group adds.""" + return len(self.capture_ids) + + +@dataclass(frozen=True, slots=True) +class SuitePlan: + """What one sync does to one suite. + + Attributes: + add: Groups to promote, in the order they were picked. + retire: ``(case name, reason)`` for every case to delete. + reviewed_over_cap: Reviewed cases beyond ``max_cases``, kept anyway. + """ + + add: tuple[CandidateGroup, ...] = () + retire: tuple[tuple[str, RetireReason], ...] = () + reviewed_over_cap: int = 0 + + @property + def added_cases(self) -> int: + """How many cases ``add`` promotes.""" + return sum(group.size for group in self.add) + + +def _hash_order(key: str) -> str: + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + +def _when(case: KeptCase) -> float: + """Sort key, oldest first; an unknown instant counts as oldest of all.""" + return case.instant.timestamp() if case.instant is not None else float("-inf") + + +def plan_suite( + kept: Sequence[KeptCase], + candidates: Sequence[CandidateGroup], + *, + max_cases: int, +) -> SuitePlan: + """Plan one suite: fill free slots route-balanced; retire stale cases only to make room. + + 1. Reviewed cases are pinned; every other case is fresh or stale. + 2. Free slots (``max_cases`` minus pinned and fresh) are filled one group at a time from + the route least represented so far (ties by route name), each route's groups in + ``sha256(key)`` order. A group is added whole, or skipped when it does not fit. + 3. A stale case whose content an added turn replays is retired outright, so no duplicate + content survives. Other stale cases leave only while the suite is over ``max_cases``: + recorded-on-another-model first, then oldest. + 4. Still over (a legacy or over-cap suite): fresh cases are trimmed from the most + represented route, oldest first. Reviewed cases never leave. + + Same inputs, same plan, on any machine. + """ + pinned = [c for c in kept if c.reviewed] + fresh = [c for c in kept if not c.reviewed and c.stale is None] + stale = [c for c in kept if not c.reviewed and c.stale is not None] + + counts: Counter[str] = Counter(c.route for c in (*pinned, *fresh)) + queues: dict[str, list[CandidateGroup]] = {} + for group in sorted(candidates, key=lambda g: _hash_order(g.key)): + queues.setdefault(group.route, []).append(group) + + slots = max(0, max_cases - len(pinned) - len(fresh)) + added: list[CandidateGroup] = [] + while slots > 0: + fitting = [r for r, queue in queues.items() if any(g.size <= slots for g in queue)] + if not fitting: + break + route = min(fitting, key=lambda r: (counts[r], r)) + queue = queues[route] + group = queue.pop(next(i for i, g in enumerate(queue) if g.size <= slots)) + added.append(group) + counts[route] += group.size + slots -= group.size + + retire: list[tuple[str, RetireReason]] = [] + replayed = frozenset(key for group in added for key in group.content_keys) + remaining: list[KeptCase] = [] + for case in stale: + if case.stale is not None and case.content_key in replayed: + retire.append((case.name, case.stale)) + else: + remaining.append(case) + + excess = len(pinned) + len(fresh) + sum(g.size for g in added) + len(remaining) - max_cases + for case in sorted(remaining, key=lambda c: (c.stale != "model", _when(c), c.name)): + if excess <= 0: + break + if case.stale is not None: + retire.append((case.name, case.stale)) + excess -= 1 + + pool = sorted(fresh, key=lambda c: (_when(c), c.name)) + while excess > 0 and pool: + route = min({c.route for c in pool}, key=lambda r: (-counts[r], r)) + victim = next(c for c in pool if c.route == route) + pool.remove(victim) + counts[route] -= 1 + retire.append((victim.name, "over_cap")) + excess -= 1 + + return SuitePlan( + add=tuple(added), + retire=tuple(retire), + reviewed_over_cap=max(0, len(pinned) - max_cases), + ) diff --git a/tests/unit/test_captures_selection.py b/tests/unit/test_captures_selection.py new file mode 100644 index 0000000..38191a9 --- /dev/null +++ b/tests/unit/test_captures_selection.py @@ -0,0 +1,141 @@ +"""``plan_suite``: which captures enter a bounded suite and which cases leave (spec §5).""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta + +import pytest + +from evalshift_cli.captures.selection import CandidateGroup, KeptCase, SuitePlan, plan_suite +from evalshift_cli.captures.staleness import StaleReason + +T0 = datetime(2026, 10, 1, tzinfo=UTC) + + +def _kept( + name: str, + *, + days: int = 0, + stale: StaleReason | None = None, + reviewed: bool = False, + route: str = "a", + key: str | None = None, +) -> KeptCase: + return KeptCase( + name=name, + instant=T0 + timedelta(days=days), + stale=stale, + reviewed=reviewed, + route=route, + content_key=key or f"k-{name}", + ) + + +def _group( + key: str, *, route: str = "a", size: int = 1, content: set[str] | None = None +) -> CandidateGroup: + ids = tuple(f"{key}-t{i}" for i in range(size)) if size > 1 else (key,) + return CandidateGroup( + key=key, + capture_ids=ids, + route=route, + content_keys=frozenset(content or {f"k-{i}" for i in ids}), + ) + + +def _added(plan: SuitePlan) -> list[str]: + return [g.key for g in plan.add] + + +def test_fills_free_slots_up_to_the_cap() -> None: + plan = plan_suite([], [_group(f"c{i}") for i in range(5)], max_cases=3) + assert plan.added_cases == 3 + assert plan.retire == () + + +def test_the_plan_is_deterministic_whatever_the_input_order() -> None: + groups = [_group(f"c{i}", route="ab"[i % 2]) for i in range(7)] + assert plan_suite([], groups, max_cases=4) == plan_suite([], groups[::-1], max_cases=4) + + +def test_the_least_represented_route_fills_first() -> None: + kept = [_kept(f"f{i}") for i in range(3)] + groups = [_group(f"a{i}") for i in range(5)] + [_group(f"b{i}", route="b") for i in range(2)] + plan = plan_suite(kept, groups, max_cases=6) + assert [g.route for g in plan.add] == ["b", "b", "a"] + + +def test_a_group_is_added_whole_or_not_at_all() -> None: + plan = plan_suite([], [_group("conv", size=3), _group("solo", route="b")], max_cases=2) + assert _added(plan) == ["solo"] + assert plan.added_cases == 1 + + +def test_stale_cases_stay_when_nothing_replaces_them() -> None: + kept = [_kept(f"s{i}", stale="age") for i in range(3)] + assert plan_suite(kept, [], max_cases=3) == SuitePlan() + + +def test_stale_cases_leave_only_to_make_room_oldest_first() -> None: + kept = [ + _kept("s1", days=0, stale="age"), + _kept("s2", days=1, stale="age"), + _kept("s3", days=2, stale="age"), + ] + plan = plan_suite(kept, [_group("new")], max_cases=3) + assert _added(plan) == ["new"] + assert plan.retire == (("s1", "age"),) + + +def test_another_model_leaves_before_old_age() -> None: + kept = [_kept("old", days=0, stale="age"), _kept("mod", days=5, stale="model")] + plan = plan_suite(kept, [_group("new")], max_cases=2) + assert plan.retire == (("mod", "model"),) + + +def test_a_stale_twin_is_replaced_even_under_the_cap() -> None: + kept = [_kept("twin", stale="age", key="k-q")] + plan = plan_suite(kept, [_group("new", content={"k-q"})], max_cases=10) + assert _added(plan) == ["new"] + assert plan.retire == (("twin", "age"),) + + +def test_reviewed_cases_are_never_retired() -> None: + kept = [_kept("r1", stale="age", reviewed=True), _kept("r2", stale="model", reviewed=True)] + plan = plan_suite(kept, [_group(f"c{i}") for i in range(3)], max_cases=1) + assert plan.add == () + assert plan.retire == () + assert plan.reviewed_over_cap == 1 + + +def test_an_over_cap_suite_is_trimmed_from_its_largest_route_oldest_first() -> None: + kept = [ + _kept("a1", days=0), + _kept("a2", days=1), + _kept("a3", days=2), + _kept("b1", days=0, route="b"), + ] + plan = plan_suite(kept, [], max_cases=2) + assert plan.retire == (("a1", "over_cap"), ("a2", "over_cap")) + + +def test_planning_twice_changes_nothing() -> None: + kept = [_kept(f"f{i}") for i in range(3)] + assert plan_suite(kept, [_group("c1"), _group("c2")], max_cases=3) == SuitePlan() + + +@pytest.mark.parametrize("max_cases", [1, 2, 3, 5, 8]) +def test_a_suite_never_ends_over_the_cap_unless_reviewed_cases_force_it(max_cases: int) -> None: + kept = [ + _kept("r", reviewed=True), + _kept("f1"), + _kept("f2", route="b"), + _kept("s1", stale="age"), + _kept("s2", stale="model", route="b"), + ] + groups = [_group("c1"), _group("c2", route="c"), _group("conv", size=2, route="b")] + plan = plan_suite(kept, groups, max_cases=max_cases) + names = [name for name, _ in plan.retire] + assert len(names) == len(set(names)) + assert "r" not in names + assert len(kept) - len(plan.retire) + plan.added_cases <= max_cases From 0505eaa64a72634f84f93c9edd1fa2c4837cc945 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:47:24 +0200 Subject: [PATCH 11/26] feat(capture): sync reads captures limits; max_age bounds the fetch Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/remote.py | 9 ++- src/evalshift_cli/cli/commands/capture.py | 94 +++++++++++++++++++++-- tests/unit/test_captures_remote.py | 11 +++ tests/unit/test_cli_capture_remote.py | 80 +++++++++++++++++++ 4 files changed, 185 insertions(+), 9 deletions(-) diff --git a/src/evalshift_cli/captures/remote.py b/src/evalshift_cli/captures/remote.py index 1cbc371..b65b0d6 100644 --- a/src/evalshift_cli/captures/remote.py +++ b/src/evalshift_cli/captures/remote.py @@ -198,6 +198,7 @@ class FetchSummary: skipped_old: int = 0 skipped_large: int = 0 skipped_malformed: int = 0 + since_label: str = "--since" def describe(self) -> str: """``fetched N capture(s) and M toolset(s) from (skipped ...)``.""" @@ -210,7 +211,7 @@ def describe(self) -> str: for n, label in ( (self.skipped_existing, "already local"), (self.skipped_promoted, "promoted"), - (self.skipped_old, "older than --since"), + (self.skipped_old, f"older than {self.since_label}"), (self.skipped_large, "over the size cap"), (self.skipped_malformed, "malformed"), ) @@ -254,6 +255,7 @@ def fetch_captures( skip_ids: Collection[str] = frozenset(), force_ids: Collection[str] = frozenset(), workers: int = 8, + since_label: str = "--since", ) -> FetchSummary: """Mirror missing toolsets and captures from ``store`` into ````. @@ -264,13 +266,14 @@ def fetch_captures( :data:`MAX_OBJECT_BYTES`. Ids in ``force_ids`` are fetched even when promoted or older than ``since`` -- the captures ``capture sync --repair`` needs to rebuild promoted cases. They still respect the already-local, size-cap and malformed-key rules. Keys that do not - match the SDK's layout are counted as malformed and ignored. + match the SDK's layout are counted as malformed and ignored. ``since_label`` is how the + summary names the ``since`` cutoff. Raises: RemoteStoreError: when listing or any download fails. Files already written stay, so a re-run resumes; nothing half-written is left behind. """ - summary = FetchSummary(store_uri=store.uri) + summary = FetchSummary(store_uri=store.uri, since_label=since_label) skip = set(skip_ids) force = set(force_ids) toolsets_dir = _absolute(toolsets_root(base)) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 855a33b..51bc20f 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -18,7 +18,8 @@ import os import re from collections.abc import Callable, Iterable, Mapping -from datetime import UTC, datetime +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Annotated, Any @@ -86,6 +87,7 @@ from evalshift_cli.traces.diff import diff_traces from evalshift_cli.traces.models import ModelCallEvent, ToolCallEvent from evalshift_cli.utils.ci_pin import check_ci_pin +from evalshift_cli.utils.duration import format_duration capture_app = typer.Typer( name="capture", @@ -140,8 +142,13 @@ def _warn_ci_pin(console: Console, config_path: Path) -> None: console.print(f"[yellow]⚠[/yellow] {escape(finding.message)}") +def _now() -> datetime: + """The one instant a sync measures capture age from (tests pin it).""" + return datetime.now(UTC) + + def _now_iso() -> str: - return datetime.now(UTC).isoformat() + return _now().isoformat() def _warn_unreadable(console: Console) -> Callable[[CaptureError], None]: @@ -442,6 +449,52 @@ def _warn_store_skipped(config_path: Path, reason: str, *, console: Console) -> ) +@dataclass(frozen=True, slots=True) +class _SyncLimits: + """``captures.max_age`` / ``captures.max_cases`` and ``defaults.source_model``, for sync. + + ``active`` is the opt-in switch (spec §1): with neither limit set, sync behaves exactly as + it did before the limits existed, whatever ``defaults.source_model`` says. + """ + + max_age: timedelta | None = None + max_cases: int | None = None + source_model: str | None = None + + @property + def active(self) -> bool: + return self.max_age is not None or self.max_cases is not None + + +def _sync_limits(config_path: Path, *, console: Console, warn: bool) -> _SyncLimits: + """Read the limits from ``config_path``; none at all when it is missing or unreadable. + + With ``warn``, an unreadable file that has a top-level ``captures`` key gets a line saying + the limits are dropped. ``capture sync`` passes it online too: the fetch step's own warning + names only ``captures.store``, so without this line sync would quietly run unbounded. + """ + if not config_path.exists(): + return _SyncLimits() + try: + cfg = load_config(config_path) + except ConfigError as exc: + reason = exc.summary + except (OSError, UnicodeDecodeError) as exc: + reason = type(exc).__name__ # its text can carry raw bytes of the file + else: + return _SyncLimits( + max_age=cfg.captures.max_age, + max_cases=cfg.captures.max_cases, + source_model=cfg.defaults.source_model, + ) + if warn and _mentions_captures(config_path): + console.print( + f"[yellow]⚠[/yellow] could not read {escape(str(config_path))} ({escape(reason)}); " + "syncing without captures.max_age / captures.max_cases.", + ) + return _SyncLimits() + + def _resolve_since(since: str | None) -> datetime | None: if since is None: return None @@ -459,6 +512,7 @@ def _fetch_into_base( since: datetime | None, console: Console, force_ids: frozenset[str] = frozenset(), + since_label: str = "--since", ) -> FetchSummary: """Mirror ``store`` into ``base`` and print the one-line summary; exit 1 on failure. @@ -468,7 +522,13 @@ def _fetch_into_base( skip_ids = _promoted_capture_ids_or_warn(base=base, console=console) try: summary = fetch_captures( - store, base=base, suite=suite, since=since, skip_ids=skip_ids, force_ids=force_ids + store, + base=base, + suite=suite, + since=since, + skip_ids=skip_ids, + force_ids=force_ids, + since_label=since_label, ) except RemoteStoreError as exc: console.print(exc.format_rich()) @@ -488,12 +548,20 @@ def _maybe_fetch( base: Path | None, console: Console, force_ids: frozenset[str] = frozenset(), + window: tuple[datetime, str] | None = None, ) -> None: - """The automatic fetch step ``sync`` and ``list`` run before reading the local mirror.""" + """The automatic fetch step ``sync`` and ``list`` run before reading the local mirror. + + ``window`` is the cutoff ``captures.max_age`` implies, and how to name it; an explicit + ``--since`` wins. + """ since_dt = _resolve_since(since) + since_label = "--since" + if since_dt is None and window is not None: + since_dt, since_label = window store = None if offline else _configured_store(config_path, console=console, strict=False) if store is None: - if since_dt is not None: + if since is not None: console.print( "[red]✗[/red] --since needs captures.store in evalshift.yaml and no --offline: " "there is nothing to fetch from.", @@ -501,7 +569,13 @@ def _maybe_fetch( raise typer.Exit(code=1) return _fetch_into_base( - store, base=base, suite=suite, since=since_dt, console=console, force_ids=force_ids + store, + base=base, + suite=suite, + since=since_dt, + since_label=since_label, + console=console, + force_ids=force_ids, ) @@ -1165,6 +1239,13 @@ def capture_sync( """Promote every capture into a suite and wire the suites: block into evalshift.yaml.""" console = Console() _check_rounds(rounds) + limits = _sync_limits(config_path, console=console, warn=True) + now = _now() + window = ( + (now - limits.max_age, f"captures.max_age ({format_duration(limits.max_age)})") + if limits.max_age is not None + else None + ) selection = _repair_selection(base=base, suite=suite, console=console) _maybe_fetch( config_path=config_path, @@ -1174,6 +1255,7 @@ def capture_sync( base=base, console=console, force_ids=selection.refetch_ids if repair else frozenset(), + window=window, ) if not repair: for line in stringified_warnings(selection): diff --git a/tests/unit/test_captures_remote.py b/tests/unit/test_captures_remote.py index d491eca..f12ba61 100644 --- a/tests/unit/test_captures_remote.py +++ b/tests/unit/test_captures_remote.py @@ -386,3 +386,14 @@ def test_force_ids_still_skip_captures_already_local(tmp_path: Path) -> None: summary = fetch_captures(store, base=tmp_path, force_ids={"cap_1"}) assert summary.skipped_existing == 1 assert store.gets == [] + + +def test_skipped_old_names_the_window_it_was_measured_against() -> None: + summary = FetchSummary( + store_uri="s3://b/p", skipped_old=3, since_label="captures.max_age (14d)" + ) + assert "3 older than captures.max_age (14d)" in summary.describe() + + +def test_skipped_old_defaults_to_the_since_flag() -> None: + assert "1 older than --since" in FetchSummary(store_uri="s3://b/p", skipped_old=1).describe() diff --git a/tests/unit/test_cli_capture_remote.py b/tests/unit/test_cli_capture_remote.py index f31c6f9..8a97af1 100644 --- a/tests/unit/test_cli_capture_remote.py +++ b/tests/unit/test_cli_capture_remote.py @@ -2,6 +2,7 @@ from __future__ import annotations +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -342,3 +343,82 @@ def test_missing_library_exits_1_naming_the_package( assert "Remote capture store" in result.stdout assert "pip install boto3" in result.stdout assert not (tmp_path / "suites").exists() + + +# --- captures limits ----------------------------------------------------------------------- + +_NOW = datetime(2026, 6, 20, tzinfo=UTC) + + +def _with_max_age(config: Path) -> Path: + config.write_text(config.read_text(encoding="utf-8") + " max_age: 14d\n", encoding="utf-8") + return config + + +def test_sync_fetches_only_inside_max_age( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, remote: FakeRemoteStore +) -> None: + remote.modified["captures/alpha/cap_1.json"] = datetime(2026, 5, 1, tzinfo=UTC) + config = _with_max_age(_write_config(tmp_path / "evalshift.yaml")) + monkeypatch.setattr(capture_module, "_now", lambda: _NOW) + + result = _invoke(["sync", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + flat = " ".join(result.stdout.split()) + assert "fetched 1 capture(s)" in flat + assert "1 older than captures.max_age (14d)" in flat + assert not (tmp_path / "captures" / "alpha" / "cap_1.json").exists() + + +def test_an_explicit_since_overrides_max_age( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, remote: FakeRemoteStore +) -> None: + remote.modified["captures/alpha/cap_1.json"] = datetime(2026, 5, 1, tzinfo=UTC) + config = _with_max_age(_write_config(tmp_path / "evalshift.yaml")) + monkeypatch.setattr(capture_module, "_now", lambda: _NOW) + + result = _invoke(["sync", "--since", "2026-04-01", "--config", str(config)], tmp_path) + + assert result.exit_code == 0, result.stdout + assert "fetched 2 capture(s)" in result.stdout + + +def test_max_age_without_a_store_is_not_a_fetch_error(tmp_path: Path) -> None: + config = _write_config(tmp_path / "evalshift.yaml", store=None) + config.write_text( + config.read_text(encoding="utf-8") + "\ncaptures:\n max_age: 14d\n", encoding="utf-8" + ) + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + + +def test_offline_sync_says_it_ignores_limits_it_cannot_read(tmp_path: Path) -> None: + config = tmp_path / "evalshift.yaml" + config.write_text("captures:\n max_age: 2w\n", encoding="utf-8") + result = _invoke(["sync", "--offline", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.stdout + assert "syncing without captures.max_age / captures.max_cases" in " ".join( + result.stdout.split() + ) + + +def test_online_sync_says_it_ignores_limits_it_cannot_read( + tmp_path: Path, remote: FakeRemoteStore +) -> None: + # An `evalshift init` config with a top-level `captures:` block, made invalid by a typo: + # the fetch step's warning names only captures.store, so the dropped limits get their own. + config = tmp_path / "evalshift.yaml" + config.write_text( + render_minimal_config(profile="model-upgrade") + "\ntypo_key: 1\n", encoding="utf-8" + ) + assert capture_module._mentions_captures(config) + result = _invoke(["sync", "--config", str(config)], tmp_path) + assert result.exit_code == 0, result.output + flat = " ".join(result.stdout.split()) + assert ( + f"could not read {config} (1 schema problem found); " + "syncing without captures.max_age / captures.max_cases." + ) in flat + assert "skipping captures.store" in flat + assert remote.gets == [] From 71cbd7f4e081f54f56134ab53daa566d95b55736 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 22:59:24 +0200 Subject: [PATCH 12/26] feat(capture): sync keeps each suite to captures.max_cases fresh cases Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 238 ++++++++++++- tests/unit/test_cli_capture_limits.py | 400 ++++++++++++++++++++++ 2 files changed, 630 insertions(+), 8 deletions(-) create mode 100644 tests/unit/test_cli_capture_limits.py diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index 51bc20f..fea4fae 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -17,7 +17,8 @@ import json import os import re -from collections.abc import Callable, Iterable, Mapping +from collections import Counter +from collections.abc import Callable, Iterable, Mapping, Sequence from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import Path @@ -41,6 +42,7 @@ promoted_suite_dir, rebuild_golden_jsonl, recover_from_messages, + retire_promoted_case, write_promoted_case, ) from evalshift_cli.captures.reader import ( @@ -72,6 +74,15 @@ stringified_content, stringified_warnings, ) +from evalshift_cli.captures.selection import CandidateGroup, KeptCase, SuitePlan, plan_suite +from evalshift_cli.captures.staleness import ( + StalenessRules, + StaleReason, + case_instant, + examples_route, + is_reviewed, + parse_instant, +) from evalshift_cli.captures.store_uri import parse_store_uri from evalshift_cli.cli.commands._suites import ( derive_suite_evaluators, @@ -83,6 +94,7 @@ from evalshift_cli.cli.commands.doctor import CONFIG_FILENAME from evalshift_cli.config.loader import ConfigError, load_config from evalshift_cli.config.models import SuiteEvaluatorsOverride +from evalshift_cli.models.matching import describe_models, model_filter_applies from evalshift_cli.suite.loader import SuiteError, load_jsonl from evalshift_cli.traces.diff import diff_traces from evalshift_cli.traces.models import ModelCallEvent, ToolCallEvent @@ -298,6 +310,98 @@ def _backfill_provenance( return out, rewritten +def _suites_with_cases(*, base: Path | None, only: str | None) -> set[str]: + """Suites under ``/suites/`` holding at least one case file (``only`` narrows).""" + root = suites_root(base) + if not root.is_dir(): + return set() + return { + path.name + for path in root.iterdir() + if path.is_dir() and any(path.glob("*.json")) and (only is None or path.name == only) + } + + +def _suite_rules( + suite: str, + limits: _SyncLimits, + *, + now: datetime, + recorded_models: list[str], + console: Console, +) -> StalenessRules: + """One suite's staleness rules, with the model guardrail applied (spec §3).""" + source = limits.source_model + if source and not model_filter_applies(recorded_models, source): + known = [m for m in recorded_models if m] + if known: + console.print( + f"[yellow]⚠[/yellow] {escape(suite)}: none of {len(known)} case(s) and " + f"capture(s) were recorded on {escape(source)} " + f"(seen: {escape(describe_models(known))}); not filtering by model — " + "check defaults.source_model", + soft_wrap=True, + ) + source = None + return StalenessRules(now=now, max_age=limits.max_age, source_model=source) + + +def _candidate_groups(pending: Sequence[PromotedCase]) -> list[CandidateGroup]: + """Group pending cases by conversation (a lone case is its own group), turn order kept.""" + turns: dict[str, list[PromotedCase]] = {} + for case in pending: + key = ( + f"conversation:{case.conversation_id}" + if case.conversation_id + else f"capture:{case.name}" + ) + turns.setdefault(key, []).append(case) + return [ + CandidateGroup( + key=key, + capture_ids=tuple(c.name for c in cases), + route=examples_route(c.example for c in cases), + content_keys=frozenset(example_content_key(c.example) for c in cases), + ) + for key, cases in turns.items() + ] + + +def _reason_counts(counts: Counter[str], *, max_age: timedelta | None) -> str: + """``2 older than 14d, 1 on another model, 3 over max_cases``; zero counts left out.""" + labels = { + "age": f"older than {format_duration(max_age)}" if max_age is not None else "too old", + "model": "on another model", + "over_cap": "over max_cases", + } + return ", ".join(f"{counts[k]} {labels[k]}" for k in ("age", "model", "over_cap") if counts[k]) + + +def _plan_line( + suite: str, + plan: SuitePlan | None, + *, + kept: int, + eligible: int, + ineligible: Counter[str], + max_age: timedelta | None, +) -> str | None: + """The per-suite summary line (spec §6); ``plan`` is ``None`` with ``max_age`` alone.""" + not_eligible = _reason_counts(ineligible, max_age=max_age) + if plan is None: + return f" {suite}: not eligible: {not_eligible}" if not_eligible else None + retired: Counter[str] = Counter(reason for _, reason in plan.retire) + detail = _reason_counts(retired, max_age=max_age) + stayed = kept - len(plan.retire) + return ( + f" {suite}: {stayed + plan.added_cases} cases — kept {stayed}, " + f"retired {len(plan.retire)}" + + (f" ({detail})" if detail else "") + + f", added {plan.added_cases} of {eligible} eligible" + + (f"; not eligible: {not_eligible}" if not_eligible else "") + ) + + def _repair_selection(*, base: Path | None, suite: str | None, console: Console) -> RepairSelection: """The promoted cases ``capture sync`` treats as repair candidates (``captures/repair.py``). @@ -1265,7 +1369,9 @@ def capture_sync( records = iter_captures(suite=suite, base=base, on_error=_warn_unreadable(console)) if repair: _warn_unrepairable(selection, records, console=console) - if not records: + if not records and not ( + limits.max_cases is not None and _suites_with_cases(base=base, only=suite) + ): where = f" for suite {suite!r}" if suite else "" console.print(f"[dim]no captures found{where} under .evalshift/captures/.[/dim]") return @@ -1294,6 +1400,7 @@ def capture_sync( repaired = 0 left_for_repair = 0 backfilled = 0 + plan_lines: list[str] = [] backed_by = selection.cases_by_capture() suites_seen: set[str] = set() # (suite, content key) -> the name of the case that owns that content, @@ -1316,20 +1423,60 @@ def capture_sync( suites_seen.add(envelope.suite) records_by_suite.setdefault(envelope.suite, []).append(record) - for suite_name in sorted(records_by_suite): + planned = set(records_by_suite) + if limits.max_cases is not None: + # An over-cap suite is trimmed even when no new capture arrived for it. + planned |= _suites_with_cases(base=base, only=suite) + suites_seen |= planned + for suite_name in sorted(planned): + suite_records = records_by_suite.get(suite_name, []) # Per-suite, not global: a conversation_id is only unique within the # suite that recorded it, same reason the grouping above is per-suite. for warning in duplicate_turn_warnings( - [r.envelope for r in records_by_suite[suite_name]], + [r.envelope for r in suite_records], ): console.print(f"[yellow]⚠[/yellow] {suite_name}: {warning}") - envelopes = {r.envelope.capture_id: r.envelope for r in records_by_suite[suite_name]} + envelopes = {r.envelope.capture_id: r.envelope for r in suite_records} kept_cases, unreadable = _load_promoted(suite_name, base=base) if unreadable is None: kept_cases, rewritten = _backfill_provenance(kept_cases, envelopes, base=base) backfilled += rewritten - built_pairs = build_conversation_examples(records_by_suite[suite_name], opts, base=base) - local_ids = {r.envelope.capture_id for r in records_by_suite[suite_name]} + if unreadable is not None and limits.active: + console.print( + f"[yellow]⚠[/yellow] {escape(suite_name)}: could not read its promoted cases " + f"({unreadable}); left it unchanged.", + ) + continue + kept_from = {c.from_capture for c in kept_cases} + stale_by_name: dict[str, StaleReason | None] = {} + rules: StalenessRules | None = None + if limits.active: + recorded = [c.source_model for c in kept_cases] + [ + capture_provenance(r.envelope)[1] + for r in suite_records + if r.envelope.capture_id not in kept_from + ] + rules = _suite_rules( + suite_name, limits, now=now, recorded_models=recorded, console=console + ) + stale_by_name = { + c.name: rules.stale_reason(case_instant(c), c.source_model) for c in kept_cases + } + # Only the planner retires a stale twin, and never a reviewed one; any other stale + # case keeps its content, or a fresh copy would be promoted beside it. + replaceable = ( + { + c.name + for c in kept_cases + if stale_by_name.get(c.name) is not None and not is_reviewed(c.example) + } + if limits.max_cases is not None + else set() + ) + ineligible: Counter[str] = Counter() + pending: list[PromotedCase] = [] + built_pairs = build_conversation_examples(suite_records, opts, base=base) + local_ids = {r.envelope.capture_id for r in suite_records} # Re-exercising an agent on the same data records duplicate captures. # Promoting both would duplicate the example, double n, and inflate # every downstream p-value/effect size. Dedup keys on the *built @@ -1337,7 +1484,12 @@ def capture_sync( # `example_content_key` for why not the envelope input_hash — and is # seeded from the cases already on disk so it spans sync runs. if not keep_duplicates: - seen_examples.update(_promoted_content_owners(suite_name, base=base, console=console)) + owners = _promoted_content_owners(suite_name, base=base, console=console) + # A replaceable stale case does not claim its content: the same question captured + # fresh replaces it (spec §5, rule 3). + seen_examples.update( + (key, owner) for key, owner in owners.items() if owner not in replaceable + ) # A repair candidate's rebuilt content belongs to the candidate, whichever capture # is built first: the disk seed above only holds its stringified content, so # without this claim a capture replaying the same turn would promote as a new case. @@ -1441,6 +1593,15 @@ def capture_sync( else: skipped_errored += 1 continue + if rules is not None and envelope.capture_id not in kept_from: + captured_at, recorded_model = capture_provenance(envelope) + reason = rules.stale_reason(parse_instant(captured_at), recorded_model) + if reason is not None: + ineligible[reason] += 1 + continue + if limits.max_cases is not None and envelope.capture_id in kept_from and not force: + skipped_existing += 1 + continue content_key = (suite_name, example_content_key(built.example)) owner = seen_examples.get(content_key) # `sync` writes the case under the capture id, so an owner of that @@ -1469,6 +1630,9 @@ def capture_sync( source_model=source_model, example=built.example, ) + if limits.max_cases is not None and envelope.capture_id not in kept_from: + pending.append(case) + continue try: write_promoted_case(case, base=base, force=force) promoted += 1 @@ -1479,6 +1643,62 @@ def capture_sync( if repaired_here: repaired += len(repaired_here) _warn_repaired_duplicates(suite_name, repaired_here, base=base, console=console) + suite_line: str | None = None + if limits.max_cases is not None: + plan = plan_suite( + [ + KeptCase( + name=c.name, + instant=case_instant(c), + stale=stale_by_name.get(c.name), + reviewed=is_reviewed(c.example), + route=examples_route([c.example]), + content_key=example_content_key(c.example), + ) + for c in kept_cases + ], + _candidate_groups(pending), + max_cases=limits.max_cases, + ) + chosen = {cid for group in plan.add for cid in group.capture_ids} + for case in pending: + if case.name not in chosen: + continue + try: + write_promoted_case(case, base=base, force=force) + except FileExistsError: + skipped_existing += 1 + continue + promoted += 1 + if case.example.generation_config: + wired_generation += 1 + for name, _reason in plan.retire: + retire_promoted_case(suite_name, name, base=base) + if plan.reviewed_over_cap: + console.print( + f"[yellow]⚠[/yellow] {escape(suite_name)}: {plan.reviewed_over_cap} " + f"reviewed case(s) over captures.max_cases {limits.max_cases}; reviewed " + "cases are never retired, so it stays over.", + ) + suite_line = _plan_line( + suite_name, + plan, + kept=len(kept_cases), + eligible=len(pending), + ineligible=ineligible, + max_age=limits.max_age, + ) + elif limits.active: + suite_line = _plan_line( + suite_name, + None, + kept=len(kept_cases), + eligible=0, + ineligible=ineligible, + max_age=limits.max_age, + ) + if suite_line is not None: + plan_lines.append(suite_line) suite_paths: dict[str, Path] = {} suite_evaluators: dict[str, SuiteEvaluatorsOverride | None] = {} @@ -1541,6 +1761,8 @@ def capture_sync( if left_for_repair: summary += f", left {left_for_repair} case(s) for --repair" console.print(summary + ".") + for line in plan_lines: + console.print(escape(line), soft_wrap=True) if not suite_paths: return diff --git a/tests/unit/test_cli_capture_limits.py b/tests/unit/test_cli_capture_limits.py new file mode 100644 index 0000000..5793d5e --- /dev/null +++ b/tests/unit/test_cli_capture_limits.py @@ -0,0 +1,400 @@ +"""``capture sync`` with ``captures.max_age`` / ``captures.max_cases`` set (bounded suites).""" + +from __future__ import annotations + +import json +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from evalshift_cli.cli.commands import capture as capture_module +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config +from evalshift_cli.cli.main import app +from evalshift_cli.suite.loader import load_jsonl +from tests.unit.fake_remote_store import TOOLSET_HEX, TOOLSET_PAYLOAD, capture_payload + +runner = CliRunner() + +WEEK_1 = "2026-06-10T12:00:00+00:00" +WEEK_3 = "2026-06-29T12:00:00+00:00" + + +def _at(monkeypatch: pytest.MonkeyPatch, day: int) -> None: + """Pin sync's clock to noon UTC on 2026-06-.""" + monkeypatch.setattr(capture_module, "_now", lambda: datetime(2026, 6, day, 12, tzinfo=UTC)) + + +def _put( + base: Path, + cid: str, + *, + suite: str = "alpha", + created_at: str = WEEK_1, + model_id: str = "claude-sonnet-5-20260301", + text: str | None = None, + conversation_id: str | None = None, + turn_index: int | None = None, +) -> None: + """Write one local capture and the shared toolset sidecar it points at.""" + sidecar = base / "toolsets" / f"{TOOLSET_HEX}.json" + sidecar.parent.mkdir(parents=True, exist_ok=True) + sidecar.write_bytes(TOOLSET_PAYLOAD) + path = base / "captures" / suite / f"{cid}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes( + capture_payload( + cid, + suite, + model_input=text if text is not None else f"question {cid}?", + created_at=created_at, + model_id=model_id, + conversation_id=conversation_id, + turn_index=turn_index, + ) + ) + + +def _config( + path: Path, + *, + max_age: str | None = "14d", + max_cases: int | None = 3, + source_model: str = "claude-sonnet-5", +) -> Path: + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + text = re.sub(r"(?m)^ source_model: .*$", f" source_model: {source_model}", text) + limits = (f" max_age: {max_age}\n" if max_age else "") + ( + f" max_cases: {max_cases}\n" if max_cases else "" + ) + if limits: + text += "\ncaptures:\n" + limits + path.write_text(text, encoding="utf-8") + return path + + +def _sync(tmp_path: Path, *args: str) -> Any: + result = runner.invoke( + app, + [ + "capture", + "sync", + "--config", + str(tmp_path / "evalshift.yaml"), + "--base", + str(tmp_path), + *args, + ], + ) + assert result.exit_code == 0, result.stdout + return result + + +def _flat(result: Any) -> str: + return " ".join(result.stdout.split()) + + +def _cases(tmp_path: Path, suite: str = "alpha") -> set[str]: + return {p.stem for p in (tmp_path / "suites" / suite).glob("*.json")} + + +def test_a_first_sync_caps_the_suite_at_max_cases( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert len(load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").examples) == 3 + assert "alpha: 3 cases — kept 0, retired 0, added 3 of 5 eligible" in _flat(result) + + +def test_syncing_again_with_nothing_new_changes_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + suite_dir = tmp_path / "suites" / "alpha" + before = {p.name: p.read_bytes() for p in suite_dir.iterdir()} + + result = _sync(tmp_path) + + assert {p.name: p.read_bytes() for p in suite_dir.iterdir()} == before + assert "alpha: 3 cases — kept 3, retired 0, added 0 of 2 eligible" in _flat(result) + + +def test_stale_cases_are_swapped_for_fresh_captures( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"old_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new_0", created_at=WEEK_3) + _put(tmp_path, "new_1", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"old_2", "new_0", "new_1"} + assert "alpha: 3 cases — kept 1, retired 2 (2 older than 14d), added 2 of 2 eligible" in _flat( + result + ) + + +def test_a_stale_suite_stays_full_when_nothing_replaces_it( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"old_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert "alpha: 3 cases — kept 3, retired 0, added 0 of 0 eligible" in _flat(result) + + +def test_cases_recorded_on_the_old_model_leave_after_a_migration( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"old_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new_0", created_at="2026-06-13T12:00:00+00:00", model_id="claude-sonnet-5-5") + _config(tmp_path / "evalshift.yaml", source_model="claude-sonnet-5-5") + _at(monkeypatch, 14) + + result = _sync(tmp_path) + + assert "new_0" in _cases(tmp_path) + assert len(_cases(tmp_path)) == 3 + assert "retired 1 (1 on another model), added 1 of 1 eligible" in _flat(result) + + +def test_a_source_model_no_capture_ran_on_filters_nothing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", source_model="gpt-9") + _at(monkeypatch, 12) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + flat = _flat(result) + assert "alpha: none of 3 case(s) and capture(s) were recorded on gpt-9" in flat + assert "(seen: claude-sonnet-5-20260301 ×3)" in flat + assert "not filtering by model — check defaults.source_model" in flat + + +def test_captures_outside_the_window_are_not_eligible( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _put(tmp_path, "old") + _put(tmp_path, "new", created_at=WEEK_3) + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"new"} + assert "added 1 of 1 eligible; not eligible: 1 older than 14d" in _flat(result) + + +def test_max_age_alone_filters_without_capping( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"new_{i}", created_at=WEEK_3) + _put(tmp_path, "old") + _config(tmp_path / "evalshift.yaml", max_cases=None) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 5 + assert "alpha: not eligible: 1 older than 14d" in _flat(result) + + +def test_an_over_cap_suite_is_trimmed_without_new_captures( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + for path in (tmp_path / "captures" / "alpha").glob("*.json"): + path.unlink() + _config(tmp_path / "evalshift.yaml", max_cases=3) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert len(load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").examples) == 3 + assert "alpha: 3 cases — kept 3, retired 2 (2 over max_cases), added 0 of 0 eligible" in _flat( + result + ) + + +def test_suite_flag_leaves_other_suites_alone( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for suite in ("alpha", "beta"): + for i in range(5): + _put(tmp_path, f"{suite}_{i}", suite=suite) + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + _config(tmp_path / "evalshift.yaml", max_cases=3) + + _sync(tmp_path, "--suite", "alpha") + + assert len(_cases(tmp_path, "alpha")) == 3 + assert len(_cases(tmp_path, "beta")) == 5 + + +def test_a_conversation_enters_whole_or_not_at_all( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for turn in range(3): + _put(tmp_path, f"conv_t{turn}", conversation_id="c1", turn_index=turn) + _put(tmp_path, "solo") + _config(tmp_path / "evalshift.yaml", max_cases=2) + _at(monkeypatch, 12) + + _sync(tmp_path) + + assert _cases(tmp_path) == {"solo"} + + +def test_the_same_question_captured_fresh_replaces_its_stale_case( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _put(tmp_path, "old", text="where is order 1?") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"new"} + assert "retired 1 (1 older than 14d), added 1 of 1 eligible" in _flat(result) + + +def test_without_limits_sync_promotes_everything_as_before( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None, source_model="gpt-9") + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 5 + assert "cases — kept" not in _flat(result) + assert "not filtering by model" not in _flat(result) + + +def test_max_age_alone_keeps_a_stale_case_from_gaining_a_twin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Without max_cases nothing is retired, so the fresh copy would sit next to the stale one. + _put(tmp_path, "old", text="where is order 1?") + _config(tmp_path / "evalshift.yaml", max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"old"} + assert len(load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").examples) == 1 + assert "skipped 1 duplicate capture(s)" in _flat(result) + + +def test_a_reviewed_stale_case_keeps_its_content( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A reviewed case is never retired, so its fresh twin must not be added beside it. + _put(tmp_path, "old", text="where is order 1?") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + case_path = tmp_path / "suites" / "alpha" / "old.json" + data = json.loads(case_path.read_text(encoding="utf-8")) + data["example"]["expected_tools"] = [{"tool_name": "lookup", "provenance": "reviewed"}] + case_path.write_text(json.dumps(data), encoding="utf-8") + _put(tmp_path, "new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"old"} + assert "alpha: 1 cases — kept 1, retired 0, added 0 of 0 eligible" in _flat(result) + + +def test_reviewed_cases_over_the_cap_stay_and_are_named( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + for case_path in (tmp_path / "suites" / "alpha").glob("*.json"): + data = json.loads(case_path.read_text(encoding="utf-8")) + data["example"]["expected_tools"] = [{"tool_name": "lookup", "provenance": "reviewed"}] + case_path.write_text(json.dumps(data), encoding="utf-8") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=2) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert ( + "alpha: 1 reviewed case(s) over captures.max_cases 2; reviewed cases are never " + "retired, so it stays over." in _flat(result) + ) + + +def test_a_suite_whose_cases_cannot_be_read_is_left_alone( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + (tmp_path / "suites" / "alpha" / "broken.json").write_text("{", encoding="utf-8") + _put(tmp_path, "cap_new") + _config(tmp_path / "evalshift.yaml", max_cases=2) + + result = runner.invoke( + app, + ["capture", "sync", "--config", str(tmp_path / "evalshift.yaml"), "--base", str(tmp_path)], + ) + + assert _cases(tmp_path) == {"cap_0", "cap_1", "cap_2", "broken"} + assert "alpha: could not read its promoted cases" in _flat(result) From 66ebaf61682d0c9a5325c160354b053cd47d9ee7 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 23:11:40 +0200 Subject: [PATCH 13/26] fix(capture): bounded sync skips unreadable suites and plans each suite once Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/cli/commands/capture.py | 35 ++++++- tests/unit/test_cli_capture_limits.py | 109 +++++++++++++++++++++- 2 files changed, 135 insertions(+), 9 deletions(-) diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index fea4fae..df9a38c 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -311,14 +311,21 @@ def _backfill_provenance( def _suites_with_cases(*, base: Path | None, only: str | None) -> set[str]: - """Suites under ``/suites/`` holding at least one case file (``only`` narrows).""" + """Directory names under ``/suites/`` holding at least one case file. + + A name is the suite's sanitised directory name (``team/support`` -> ``team_support``), not + necessarily the suite name its cases record. ``only`` (a suite name) narrows to its directory. + """ root = suites_root(base) if not root.is_dir(): return set() + only_dir = promoted_suite_dir(only, base=base).name if only is not None else None return { path.name for path in root.iterdir() - if path.is_dir() and any(path.glob("*.json")) and (only is None or path.name == only) + if path.is_dir() + and any(path.glob("*.json")) + and (only_dir is None or path.name == only_dir) } @@ -1425,8 +1432,10 @@ def capture_sync( planned = set(records_by_suite) if limits.max_cases is not None: - # An over-cap suite is trimmed even when no new capture arrived for it. - planned |= _suites_with_cases(base=base, only=suite) + # An over-cap suite is trimmed even when no new capture arrived for it. A directory + # already planned under the suite name its captures record is not planned again. + captured_dirs = {promoted_suite_dir(name, base=base).name for name in records_by_suite} + planned |= _suites_with_cases(base=base, only=suite) - captured_dirs suites_seen |= planned for suite_name in sorted(planned): suite_records = records_by_suite.get(suite_name, []) @@ -1438,6 +1447,15 @@ def capture_sync( console.print(f"[yellow]⚠[/yellow] {suite_name}: {warning}") envelopes = {r.envelope.capture_id: r.envelope for r in suite_records} kept_cases, unreadable = _load_promoted(suite_name, base=base) + recorded_names = {c.suite for c in kept_cases} + if not suite_records and len(recorded_names) == 1: + # A suite planned from its directory alone goes by the name its cases record + # (``team/support``, not its directory ``team_support``), as its suites entry does. + (recorded_name,) = recorded_names + if promoted_suite_dir(recorded_name, base=base).name == suite_name: + suites_seen.discard(suite_name) + suites_seen.add(recorded_name) + suite_name = recorded_name if unreadable is None: kept_cases, rewritten = _backfill_provenance(kept_cases, envelopes, base=base) backfilled += rewritten @@ -1446,6 +1464,9 @@ def capture_sync( f"[yellow]⚠[/yellow] {escape(suite_name)}: could not read its promoted cases " f"({unreadable}); left it unchanged.", ) + # Not rebuilt either: its golden.jsonl and suites entry stay as they are, and the + # other suites still sync. + suites_seen.discard(suite_name) continue kept_from = {c.from_capture for c in kept_cases} stale_by_name: dict[str, StaleReason | None] = {} @@ -1473,6 +1494,7 @@ def capture_sync( if limits.max_cases is not None else set() ) + replaceable_from = {c.from_capture for c in kept_cases if c.name in replaceable} ineligible: Counter[str] = Counter() pending: list[PromotedCase] = [] built_pairs = build_conversation_examples(suite_records, opts, base=base) @@ -1610,7 +1632,10 @@ def capture_sync( if not keep_duplicates and owner is not None and owner != envelope.capture_id: skipped_duplicates += 1 continue - seen_examples.setdefault(content_key, envelope.capture_id) + if envelope.capture_id not in replaceable_from: + # A replaceable stale case refreshed under --force still leaves its content + # to a fresh twin, whichever capture id sorts first. + seen_examples.setdefault(content_key, envelope.capture_id) for warning in built.warnings: console.print(f"[yellow]⚠[/yellow] {envelope.capture_id}: {warning}") captured_at, source_model = capture_provenance(envelope) diff --git a/tests/unit/test_cli_capture_limits.py b/tests/unit/test_cli_capture_limits.py index 5793d5e..8fa5075 100644 --- a/tests/unit/test_cli_capture_limits.py +++ b/tests/unit/test_cli_capture_limits.py @@ -387,14 +387,115 @@ def test_a_suite_whose_cases_cannot_be_read_is_left_alone( _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) _at(monkeypatch, 12) _sync(tmp_path) + golden = tmp_path / "suites" / "alpha" / "golden.jsonl" + before = golden.read_bytes() (tmp_path / "suites" / "alpha" / "broken.json").write_text("{", encoding="utf-8") _put(tmp_path, "cap_new") _config(tmp_path / "evalshift.yaml", max_cases=2) - result = runner.invoke( - app, - ["capture", "sync", "--config", str(tmp_path / "evalshift.yaml"), "--base", str(tmp_path)], - ) + result = _sync(tmp_path) assert _cases(tmp_path) == {"cap_0", "cap_1", "cap_2", "broken"} + assert golden.read_bytes() == before assert "alpha: could not read its promoted cases" in _flat(result) + + +def test_a_stray_unreadable_suite_does_not_stop_the_others( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + for path in (tmp_path / "captures" / "alpha").glob("*.json"): + path.unlink() + stray = tmp_path / "suites" / "aaa" / "notes.json" + stray.parent.mkdir(parents=True) + stray.write_text('{"note": "not a case"}', encoding="utf-8") + _config(tmp_path / "evalshift.yaml", max_cases=3) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + assert len(load_jsonl(tmp_path / "suites" / "alpha" / "golden.jsonl").examples) == 3 + assert not (tmp_path / "suites" / "aaa" / "golden.jsonl").exists() + flat = _flat(result) + assert "aaa: could not read its promoted cases" in flat + assert "✓ promoted 0 capture(s) into 1 suite(s)" in flat + assert "alpha: 3 cases — kept 3, retired 2 (2 over max_cases)" in flat + assert " aaa:" not in (tmp_path / "evalshift.yaml").read_text(encoding="utf-8") + + +def _put_team_support(base: Path, cid: str) -> None: + """A capture of suite ``team/support``, in the directory the SDK sanitises it to.""" + sidecar = base / "toolsets" / f"{TOOLSET_HEX}.json" + sidecar.parent.mkdir(parents=True, exist_ok=True) + sidecar.write_bytes(TOOLSET_PAYLOAD) + path = base / "captures" / "team_support" / f"{cid}.json" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes( + capture_payload( + cid, + "team/support", + model_input=f"question {cid}?", + created_at=WEEK_1, + model_id="claude-sonnet-5-20260301", + ) + ) + + +def test_a_suite_named_with_a_slash_is_planned_once( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(4): + _put_team_support(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put_team_support(tmp_path, "cap_new") + + result = _sync(tmp_path) + + assert len(_cases(tmp_path, "team_support")) == 3 + flat = _flat(result) + assert flat.count("cases — kept") == 1 + assert "team/support: 3 cases — kept 3" in flat + config = (tmp_path / "evalshift.yaml").read_text(encoding="utf-8") + assert "team_support:" not in config + assert config.count("team/support") >= 1 + + +def test_a_suite_named_with_a_slash_keeps_its_name_without_captures( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(5): + _put_team_support(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + for path in (tmp_path / "captures" / "team_support").glob("*.json"): + path.unlink() + _config(tmp_path / "evalshift.yaml", max_cases=3) + + result = _sync(tmp_path, "--suite", "team/support") + + assert len(_cases(tmp_path, "team_support")) == 3 + assert "team/support: 3 cases — kept 3, retired 2 (2 over max_cases)" in _flat(result) + assert "team_support:" not in (tmp_path / "evalshift.yaml").read_text(encoding="utf-8") + + +@pytest.mark.parametrize("old", ["a_old", "z_old"]) +def test_force_replaces_a_stale_twin_whatever_the_capture_order( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, old: str +) -> None: + _put(tmp_path, old, text="where is order 1?") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "b_new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + + _sync(tmp_path, "--force") + + assert _cases(tmp_path) == {"b_new"} From d87cd559d18b1c891daf4366339505e32765efe0 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 23:22:06 +0200 Subject: [PATCH 14/26] feat(doctor,compare): warn when a captured suite is stale or over its cap Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/freshness.py | 140 ++++++++++++++++++++++ src/evalshift_cli/cli/commands/compare.py | 47 +++++++- src/evalshift_cli/cli/commands/doctor.py | 60 ++++++++++ tests/unit/test_captures_freshness.py | 120 +++++++++++++++++++ tests/unit/test_compare_command.py | 70 +++++++++++ tests/unit/test_doctor.py | 70 +++++++++++ 6 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 src/evalshift_cli/captures/freshness.py create mode 100644 tests/unit/test_captures_freshness.py diff --git a/src/evalshift_cli/captures/freshness.py b/src/evalshift_cli/captures/freshness.py new file mode 100644 index 0000000..685347b --- /dev/null +++ b/src/evalshift_cli/captures/freshness.py @@ -0,0 +1,140 @@ +"""How fresh a committed capture suite is, for ``doctor`` and ``compare`` (spec §7). + +Reads the promoted case files committed beside ``golden.jsonl`` -- the only place a case's +capture time and model live -- so a team that only runs CI still learns its suite went stale. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from pathlib import Path +from typing import Literal + +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.staleness import case_instant +from evalshift_cli.models.matching import describe_models, model_filter_applies, same_model +from evalshift_cli.utils.duration import format_duration + +FindingKind = Literal["unreadable", "stale_age", "stale_model", "no_source_model_case", "over_cap"] + + +@dataclass(frozen=True, slots=True) +class FreshnessFinding: + """One thing that is stale about a suite, worded for a doctor row or a compare line.""" + + kind: FindingKind + message: str + + +@dataclass(frozen=True, slots=True) +class SuiteFreshness: + """A suite's case count, capture-time span and findings.""" + + suite: str + cases: int + newest: datetime | None + oldest: datetime | None + findings: tuple[FreshnessFinding, ...] + + def describe(self, now: datetime) -> str: + """``alpha: 200 cases, captured 2d–12d ago`` (the span left out when unknown).""" # noqa: RUF002 + text = f"{self.suite}: {self.cases} cases" + if self.newest is not None and self.oldest is not None: + text += f", captured {_age(now, self.newest)}–{_age(now, self.oldest)} ago" # noqa: RUF001 — an age span + return text + + +def _age(now: datetime, then: datetime) -> str: + return f"{max(0, (now - then).days)}d" + + +def suite_freshness( + suite: str, + suite_dir: Path, + *, + max_age: timedelta | None, + max_cases: int | None, + source_model: str | None, + now: datetime, +) -> SuiteFreshness | None: + """Read ``suite_dir``'s case files and report what is stale; ``None`` when there are none. + + Never raises: an unreadable case file becomes one ``unreadable`` finding and is left out + of every count. + + Args: + suite: The suite's name, which every finding message starts with. + suite_dir: The directory holding the suite's promoted case files. + max_age: ``captures.max_age``; ``None`` turns the age finding off. + max_cases: ``captures.max_cases``; ``None`` turns the over-cap finding off. + source_model: The model cases should have been recorded on; ``None`` turns the + model findings off. + now: The instant ages are measured from. + + Returns: + The suite's freshness, or ``None`` when ``suite_dir`` holds no case files. + """ + paths = sorted(suite_dir.glob("*.json")) if suite_dir.is_dir() else [] + cases: list[PromotedCase] = [] + unreadable = 0 + for path in paths: + try: + cases.append(PromotedCase.model_validate_json(path.read_text(encoding="utf-8"))) + except (OSError, ValueError): # ValidationError and UnicodeDecodeError are ValueErrors + unreadable += 1 + if not cases and not unreadable: + return None + instants = [i for i in (case_instant(c) for c in cases) if i is not None] + newest = max(instants, default=None) + oldest = min(instants, default=None) + findings: list[FreshnessFinding] = [] + if unreadable: + findings.append( + FreshnessFinding( + "unreadable", + f"{suite}: {unreadable} case file(s) could not be read; the counts leave them out", + ) + ) + if max_age is not None and newest is not None and newest < now - max_age: + findings.append( + FreshnessFinding( + "stale_age", + f"{suite}: newest case captured {_age(now, newest)} ago " + f"(captures.max_age {format_duration(max_age)}); " + "refresh with evalshift capture sync", + ) + ) + models = [c.source_model for c in cases] + if source_model and any(models): + if model_filter_applies(models, source_model): + other = sum(1 for m in models if m and not same_model(m, source_model)) + if other: + findings.append( + FreshnessFinding( + "stale_model", + f"{suite}: {other} of {len(cases)} cases recorded on another model " + f"than {source_model}; evalshift capture sync replaces them as new " + "captures arrive", + ) + ) + else: + findings.append( + FreshnessFinding( + "no_source_model_case", + f"{suite}: no case recorded on {source_model} " + f"(seen: {describe_models(models)}); check defaults.source_model, " + "or record new captures", + ) + ) + if max_cases is not None and len(cases) > max_cases: + findings.append( + FreshnessFinding( + "over_cap", + f"{suite}: {len(cases)} cases, over captures.max_cases {max_cases}; " + "evalshift capture sync trims it", + ) + ) + return SuiteFreshness( + suite=suite, cases=len(cases), newest=newest, oldest=oldest, findings=tuple(findings) + ) diff --git a/src/evalshift_cli/cli/commands/compare.py b/src/evalshift_cli/cli/commands/compare.py index cb5a8ba..a4edfeb 100644 --- a/src/evalshift_cli/cli/commands/compare.py +++ b/src/evalshift_cli/cli/commands/compare.py @@ -24,15 +24,18 @@ import webbrowser from collections.abc import Sequence from dataclasses import dataclass +from datetime import UTC, datetime from pathlib import Path from typing import Annotated, Literal import typer from rich.console import Console, Group, RenderableType from rich.live import Live +from rich.markup import escape from rich.text import Text from evalshift_cli.analysis.statistics import ComparisonResult +from evalshift_cli.captures.freshness import suite_freshness from evalshift_cli.cli.commands._star import print_star_prompt from evalshift_cli.cli.commands._suites import ( SUITE_FILENAME, @@ -62,7 +65,7 @@ ) from evalshift_cli.cli.commands.report import run_report from evalshift_cli.config.loader import ConfigError, load_config -from evalshift_cli.config.models import EvaluatorsConfig +from evalshift_cli.config.models import EvalShiftConfig, EvaluatorsConfig from evalshift_cli.evaluators.keys import LLM_JUDGE_KIND, SEMANTIC_EVALUATOR_NAME, EvaluatorKeyGap from evalshift_cli.evaluators.tool_loader import ToolLoaderError from evalshift_cli.hosted.push import PushError, push_local_run @@ -351,6 +354,42 @@ def _warn_if_legacy_name(ctx: typer.Context) -> None: ) +def _freshness_lines( + cfg: EvalShiftConfig, suite_name: str | None, suite_path: Path, source_model: str +) -> list[str]: + """``⚠`` lines for a stale captured suite (spec §7). + + compare drops doctor's ``warn`` rows, so it asks for the findings itself. Only for a suite + selected by name whose entry is ``source: captured``, and only with a capture limit set. + + Args: + cfg: The loaded configuration. + suite_name: The ``--suite-name`` selected, or ``None`` for a suite given by path. + suite_path: The resolved ``golden.jsonl``; its case files sit beside it. + source_model: The source model this compare runs, which cases should be recorded on. + + Returns: + One Rich-markup line per finding; empty when there is nothing to say. + """ + captures = cfg.captures + if suite_name is None or (captures.max_age is None and captures.max_cases is None): + return [] + entry = cfg.suites.get(suite_name) + if entry is None or entry.source != "captured": + return [] + fresh = suite_freshness( + suite_name, + suite_path.parent, + max_age=captures.max_age, + max_cases=captures.max_cases, + source_model=source_model, + now=datetime.now(UTC), + ) + if fresh is None: + return [] + return [f"[yellow]⚠[/yellow] {escape(f.message)}" for f in fresh.findings] + + def compare_command( ctx: typer.Context, source: Annotated[ @@ -498,6 +537,7 @@ def compare_command( except SuiteError as exc: console.print(exc.format_rich()) raise typer.Exit(code=1) from exc + freshness = _freshness_lines(cfg, suite_name, suite_path, source_model) run_slug = derive_suite_slug(suite_name=suite_name, suite_path=suite_path) @@ -717,6 +757,11 @@ def _on_progress(ev: ProgressEvent) -> None: for line in evaluate_result.tool_free_warnings: console.print(line) console.print() + # And again: a stale suite gates on ground truth production no longer produces. + if freshness: + for line in freshness: + console.print(line, soft_wrap=True) + console.print() collapsed_note = format_collapsed_slices(analyze_result.collapsed_slices) if collapsed_note is not None: console.print(collapsed_note, style="dim") diff --git a/src/evalshift_cli/cli/commands/doctor.py b/src/evalshift_cli/cli/commands/doctor.py index d84d9a2..8d2ed19 100644 --- a/src/evalshift_cli/cli/commands/doctor.py +++ b/src/evalshift_cli/cli/commands/doctor.py @@ -30,6 +30,7 @@ import sys from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass +from datetime import UTC, datetime from importlib.metadata import PackageNotFoundError, version from pathlib import Path from types import ModuleType @@ -41,6 +42,7 @@ from rich.table import Table from evalshift_cli import __version__ +from evalshift_cli.captures.freshness import suite_freshness from evalshift_cli.captures.remote import RemoteStoreError, open_store from evalshift_cli.captures.store_uri import parse_store_uri from evalshift_cli.captures.toolset import EMPTY_TOOLSET_FINGERPRINT, fingerprint_tools @@ -67,6 +69,10 @@ JUDGE_FAMILY_CHECK: Final = "judge family" # Row name for the judge / embedding key check (one row per keyless model). EVALUATOR_KEYS_CHECK: Final = "evaluator keys" +# Row names for the bounded-suite checks: one ``suite freshness`` row per captured suite (or +# per finding), and a ``captures limits`` nudge when a bucket is set without limits. +SUITE_FRESHNESS_CHECK: Final = "suite freshness" +CAPTURES_LIMITS_CHECK: Final = "captures limits" # The capture SDK: a declared dependency of this package, and the owner of the # import name ``evalshift`` (this package imports as ``evalshift_cli``). SDK_DISTRIBUTION: Final = "evalshift-sdk" @@ -123,6 +129,7 @@ def run_checks(cwd: Path, env: Mapping[str, str]) -> list[CheckResult]: results.extend(_api_key_check(env, aliases) for aliases in PROVIDER_KEYS) results.append(_config_check(cwd)) results.extend(_captures_store_check(cwd)) + results.extend(_suite_freshness_checks(cwd)) results.extend(_tool_consistency_checks(cwd)) results.extend(_judge_family_checks(cwd)) results.extend(_evaluator_key_checks(cwd, env)) @@ -291,6 +298,59 @@ def _captures_store_check(cwd: Path) -> list[CheckResult]: return [CheckResult(name="captures.store", status="ok", detail=f"{uri} reachable")] +def _suite_freshness_checks(cwd: Path) -> list[CheckResult]: + """Bounded-suite rows (spec §7). Never ``fail``: freshness is advice. + + ``captures limits`` warns when ``captures.store`` is set without ``max_age`` / + ``max_cases`` -- every sync then adds to the suite. With a limit set, each + ``source: captured`` suite gets one ``ok`` row, or one ``warn`` row per finding. + """ + cfg_path = cwd / CONFIG_FILENAME + if not cfg_path.exists(): + return [] + try: + cfg = load_config(cfg_path) + except ConfigError: + return [] + captures = cfg.captures + if captures.max_age is None and captures.max_cases is None: + if captures.store is None: + return [] + return [ + CheckResult( + name=CAPTURES_LIMITS_CHECK, + status="warn", + detail="suites grow with every capture sync; " + "set captures.max_cases and captures.max_age", + ) + ] + now = datetime.now(UTC) + rows: list[CheckResult] = [] + for name, path in _named_suite_paths(cwd, cfg): + entry = cfg.suites.get(name) + if entry is None or entry.source != "captured": + continue + fresh = suite_freshness( + name, + path.parent, + max_age=captures.max_age, + max_cases=captures.max_cases, + source_model=cfg.defaults.source_model, + now=now, + ) + if fresh is None: + continue + if not fresh.findings: + rows.append( + CheckResult(name=SUITE_FRESHNESS_CHECK, status="ok", detail=fresh.describe(now)) + ) + rows.extend( + CheckResult(name=SUITE_FRESHNESS_CHECK, status="warn", detail=f.message) + for f in fresh.findings + ) + return rows + + def _named_suite_paths(cwd: Path, cfg: EvalShiftConfig) -> list[tuple[str, Path]]: """Return ``(name, path)`` for every golden-suite file the toolset check should inspect. diff --git a/tests/unit/test_captures_freshness.py b/tests/unit/test_captures_freshness.py new file mode 100644 index 0000000..fbbca26 --- /dev/null +++ b/tests/unit/test_captures_freshness.py @@ -0,0 +1,120 @@ +"""``suite_freshness``: what doctor and compare say about a committed capture suite.""" + +from __future__ import annotations + +from datetime import UTC, datetime, timedelta +from pathlib import Path +from typing import Any + +from evalshift_cli.captures.freshness import SuiteFreshness, suite_freshness +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.promote import write_promoted_case +from evalshift_cli.suite.models import SuiteExample + +NOW = datetime(2026, 10, 16, 12, tzinfo=UTC) + + +def _write(base: Path, i: int, *, days: int, model: str = "claude-sonnet-5") -> None: + write_promoted_case( + PromotedCase( + name=f"cap_{i}", + suite="alpha", + from_capture=f"cap_{i}", + captured_at=(NOW - timedelta(days=days)).isoformat(), + source_model=model, + example=SuiteExample(id=f"cap_{i}", inputs={"input": f"q{i}"}, tools=[]), + ), + base=base, + ) + + +def _check(base: Path, **limits: Any) -> SuiteFreshness: + options: dict[str, Any] = { + "max_age": timedelta(days=14), + "max_cases": 5, + "source_model": "claude-sonnet-5", + **limits, + } + fresh = suite_freshness("alpha", base / "suites" / "alpha", now=NOW, **options) + assert fresh is not None + return fresh + + +def test_no_case_files_means_nothing_to_report(tmp_path: Path) -> None: + assert ( + suite_freshness( + "alpha", + tmp_path / "suites" / "alpha", + max_age=timedelta(days=14), + max_cases=5, + source_model="claude-sonnet-5", + now=NOW, + ) + is None + ) + + +def test_a_fresh_suite_has_no_findings(tmp_path: Path) -> None: + _write(tmp_path, 0, days=2) + _write(tmp_path, 1, days=12) + fresh = _check(tmp_path) + assert fresh.findings == () + assert fresh.describe(NOW) == "alpha: 2 cases, captured 2d–12d ago" # noqa: RUF001 — doctor copy + + +def test_a_suite_past_max_age_says_how_to_refresh(tmp_path: Path) -> None: + _write(tmp_path, 0, days=23) + [finding] = _check(tmp_path).findings + assert finding.kind == "stale_age" + assert finding.message == ( + "alpha: newest case captured 23d ago (captures.max_age 14d); " + "refresh with evalshift capture sync" + ) + + +def test_cases_on_another_model_are_counted(tmp_path: Path) -> None: + _write(tmp_path, 0, days=1) + _write(tmp_path, 1, days=1, model="claude-sonnet-4-5") + [finding] = _check(tmp_path).findings + assert finding.kind == "stale_model" + assert finding.message.startswith( + "alpha: 1 of 2 cases recorded on another model than claude-sonnet-5" + ) + + +def test_no_case_on_the_source_model_is_its_own_finding(tmp_path: Path) -> None: + _write(tmp_path, 0, days=1, model="gpt-5") + [finding] = _check(tmp_path).findings + assert finding.kind == "no_source_model_case" + assert "no case recorded on claude-sonnet-5 (seen: gpt-5 ×1)" in finding.message + + +def test_a_suite_over_its_cap_is_reported(tmp_path: Path) -> None: + for i in range(3): + _write(tmp_path, i, days=1) + [finding] = _check(tmp_path, max_cases=2).findings + assert finding.message == ( + "alpha: 3 cases, over captures.max_cases 2; evalshift capture sync trims it" + ) + + +def test_an_unreadable_case_file_is_a_finding_not_a_crash(tmp_path: Path) -> None: + _write(tmp_path, 0, days=1) + (tmp_path / "suites" / "alpha" / "broken.json").write_text("{", encoding="utf-8") + fresh = _check(tmp_path) + assert fresh.cases == 1 + assert [f.kind for f in fresh.findings] == ["unreadable"] + + +def test_a_legacy_case_ages_from_its_promotion(tmp_path: Path) -> None: + write_promoted_case( + PromotedCase( + name="old", + suite="alpha", + from_capture="old", + promoted_at=(NOW - timedelta(days=30)).isoformat(), + example=SuiteExample(id="old", inputs={"input": "q"}, tools=[]), + ), + base=tmp_path, + ) + assert [f.kind for f in _check(tmp_path).findings] == ["stale_age"] diff --git a/tests/unit/test_compare_command.py b/tests/unit/test_compare_command.py index 3469a6f..44321c6 100644 --- a/tests/unit/test_compare_command.py +++ b/tests/unit/test_compare_command.py @@ -20,12 +20,15 @@ from typer.testing import CliRunner from evalshift_cli.analysis.statistics import ComparisonResult +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.promote import rebuild_golden_jsonl, write_promoted_case from evalshift_cli.cli.commands.analyze import AnalyzeResult from evalshift_cli.cli.commands.analyze import run_analyze as _real_run_analyze from evalshift_cli.cli.commands.compare import ( _bar, _compose_verdict, _evaluator_family_summary, + _freshness_lines, ) from evalshift_cli.cli.commands.doctor import CheckResult from evalshift_cli.cli.main import app @@ -38,6 +41,7 @@ StructuralEvaluatorConfig, ) from evalshift_cli.models.client import CompletionResult, ModelClient +from evalshift_cli.suite.models import SuiteExample runner = CliRunner() @@ -264,6 +268,46 @@ async def fake_complete(self: ModelClient, **kwargs: Any) -> CompletionResult: """A realistic hosted run URL: 102 characters, well past the 80-column fold.""" +def _captured_cases(tmp_path: Path, suite: str, n: int) -> Path: + for i in range(n): + write_promoted_case( + PromotedCase( + name=f"ex{i}", + suite=suite, + from_capture=f"cap_{i}", + example=SuiteExample(id=f"ex{i}", inputs={"name": f"User{i}"}, tools=[]), + ), + base=tmp_path / ".evalshift", + ) + return rebuild_golden_jsonl(tmp_path / ".evalshift" / "suites" / suite) + + +class TestFreshnessLines: + def _cfg(self, captures: dict[str, object]) -> EvalShiftConfig: + return EvalShiftConfig( + prompts=[ + {"id": "p", "detection": "manual", "content": "{name}", "variables": ["name"]} + ], + captures=captures, + suites={ + "greet": {"source": "captured", "path": ".evalshift/suites/greet/golden.jsonl"} + }, + ) + + def test_an_over_cap_suite_gets_a_line(self, tmp_path: Path) -> None: + golden = _captured_cases(tmp_path, "greet", 3) + [line] = _freshness_lines(self._cfg({"max_cases": 2}), "greet", golden, "gemini-2.5-flash") + assert "greet: 3 cases, over captures.max_cases 2" in line + + def test_no_lines_without_limits(self, tmp_path: Path) -> None: + golden = _captured_cases(tmp_path, "greet", 3) + assert _freshness_lines(self._cfg({}), "greet", golden, "gemini-2.5-flash") == [] + + def test_no_lines_for_a_suite_given_by_path(self, tmp_path: Path) -> None: + golden = _captured_cases(tmp_path, "greet", 3) + assert _freshness_lines(self._cfg({"max_cases": 2}), None, golden, "gemini-2.5-flash") == [] + + class TestEndToEnd: def test_all_runs_full_pipeline( self, @@ -430,6 +474,32 @@ def test_all_prints_the_tool_free_warning_it_scored_quietly( ) in flat assert flat.count("checked nothing") == 1 + def test_compare_warns_when_the_suite_is_over_its_cap( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + _scaffold(tmp_path) + config_path = tmp_path / "evalshift.yaml" + config_path.write_text( + config_path.read_text(encoding="utf-8").rstrip() + + "\n" + + " captures:\n max_cases: 2\n" + + " suites:\n greet:\n source: captured\n" + + " path: .evalshift/suites/greet/golden.jsonl\n", + encoding="utf-8", + ) + _captured_cases(tmp_path, "greet", 3) + _patch_client(monkeypatch) + monkeypatch.chdir(tmp_path) + + result = runner.invoke(app, ["compare", "--suite-name", "greet", "--yes"]) + + assert result.exit_code == 0, result.output + flat = " ".join(result.output.split()) + assert "greet: 3 cases, over captures.max_cases 2; evalshift capture sync trims it" in flat + assert flat.count("over captures.max_cases") == 1 + def test_all_pushes_before_gate_exit( self, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_doctor.py b/tests/unit/test_doctor.py index a4ba62e..0234f5e 100644 --- a/tests/unit/test_doctor.py +++ b/tests/unit/test_doctor.py @@ -12,6 +12,7 @@ import json from collections.abc import Iterator, Mapping +from datetime import UTC, datetime, timedelta from importlib.metadata import PackageNotFoundError from pathlib import Path from types import ModuleType @@ -19,15 +20,19 @@ import pytest from typer.testing import CliRunner +from evalshift_cli.captures.models import PromotedCase +from evalshift_cli.captures.promote import rebuild_golden_jsonl, write_promoted_case from evalshift_cli.captures.remote import ObjectInfo, RemoteStoreUnavailable from evalshift_cli.captures.toolset import fingerprint_tools from evalshift_cli.cli.commands import doctor as doctor_module from evalshift_cli.cli.commands.doctor import ( + CAPTURES_LIMITS_CHECK, CONFIG_FILENAME, EVALUATOR_KEYS_CHECK, JUDGE_FAMILY_CHECK, PROVIDER_KEYS, SDK_DISTRIBUTION, + SUITE_FRESHNESS_CHECK, CheckResult, _sdk_check, _tool_consistency_checks, @@ -38,6 +43,7 @@ from evalshift_cli.evaluators.base import EvalRecord from evalshift_cli.evaluators.failures import BROKEN_HARNESS_CAUSES from evalshift_cli.evaluators.tool_selection import KIND_CONFORMANCE, KIND_DIVERGENCE +from evalshift_cli.suite.models import SuiteExample runner = CliRunner() @@ -917,3 +923,67 @@ def _raise(parsed: object) -> object: result = runner.invoke(app, ["doctor"], env={"COLUMNS": "200"}) assert result.exit_code == 0 assert 'pip install "evalshift[gcs]"' in result.stdout + + +class TestSuiteFreshnessCheck: + def _project( + self, tmp_path: Path, *, captures: str, ages_days: list[int], model: str = "claude-sonnet-5" + ) -> None: + (tmp_path / CONFIG_FILENAME).write_text( + "prompts:\n - id: p\n detection: manual\n content: '{input}'\n" + " variables: [input]\n" + "defaults:\n source_model: claude-sonnet-5\n" + f"{captures}" + "suites:\n alpha:\n source: captured\n" + " path: .evalshift/suites/alpha/golden.jsonl\n", + encoding="utf-8", + ) + now = datetime.now(UTC) + for i, days in enumerate(ages_days): + write_promoted_case( + PromotedCase( + name=f"cap_{i}", + suite="alpha", + from_capture=f"cap_{i}", + captured_at=(now - timedelta(days=days)).isoformat(), + source_model=model, + example=SuiteExample(id=f"cap_{i}", inputs={"input": f"q{i}"}, tools=[]), + ), + base=tmp_path / ".evalshift", + ) + rebuild_golden_jsonl(tmp_path / ".evalshift" / "suites" / "alpha") + + def _rows(self, tmp_path: Path, name: str = SUITE_FRESHNESS_CHECK) -> list[CheckResult]: + return [r for r in run_checks(cwd=tmp_path, env=_empty_env()) if r.name == name] + + def test_a_fresh_suite_is_ok(self, tmp_path: Path) -> None: + self._project( + tmp_path, captures="captures:\n max_age: 14d\n max_cases: 5\n", ages_days=[2, 12] + ) + [row] = self._rows(tmp_path) + assert row.status == "ok" + assert row.detail == "alpha: 2 cases, captured 2d–12d ago" # noqa: RUF001 — doctor copy + + def test_a_stale_suite_warns(self, tmp_path: Path) -> None: + self._project(tmp_path, captures="captures:\n max_age: 14d\n", ages_days=[23, 30]) + [row] = self._rows(tmp_path) + assert row.status == "warn" + assert "newest case captured 23d ago (captures.max_age 14d)" in row.detail + + def test_no_rows_without_limits(self, tmp_path: Path) -> None: + self._project(tmp_path, captures="", ages_days=[30]) + assert self._rows(tmp_path) == [] + assert self._rows(tmp_path, CAPTURES_LIMITS_CHECK) == [] + + def test_a_bucket_without_limits_gets_a_nudge( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from tests.unit.fake_remote_store import populated_store + + self._project(tmp_path, captures="captures:\n store: s3://acme-evals/p\n", ages_days=[1]) + monkeypatch.setattr(doctor_module, "open_store", lambda parsed: populated_store("cap_1")) + [row] = self._rows(tmp_path, CAPTURES_LIMITS_CHECK) + assert row.status == "warn" + assert row.detail == ( + "suites grow with every capture sync; set captures.max_cases and captures.max_age" + ) From 0e4282bcf02266e9d4b9b950d5b0e8bccdeaac3f Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 23:30:43 +0200 Subject: [PATCH 15/26] fix(doctor,compare): freshness for the auto-selected suite; captures limits per spec Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/staleness.py | 5 +- src/evalshift_cli/cli/commands/_suites.py | 46 ++++++++++++++--- src/evalshift_cli/cli/commands/compare.py | 21 +++++--- src/evalshift_cli/cli/commands/doctor.py | 25 +++++---- tests/unit/test_captures_staleness.py | 4 +- tests/unit/test_compare_command.py | 63 ++++++++++++++++++++--- tests/unit/test_doctor.py | 33 ++++++++++++ tests/unit/test_suites_helpers.py | 26 ++++++++++ 8 files changed, 190 insertions(+), 33 deletions(-) diff --git a/src/evalshift_cli/captures/staleness.py b/src/evalshift_cli/captures/staleness.py index 8f065fb..4e7f9ee 100644 --- a/src/evalshift_cli/captures/staleness.py +++ b/src/evalshift_cli/captures/staleness.py @@ -32,7 +32,10 @@ def parse_instant(text: str) -> datetime | None: return None if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=UTC) - return parsed.astimezone(UTC) + try: + return parsed.astimezone(UTC) + except OverflowError: # an offset that pushes it past datetime.min / max + return None def case_instant(case: PromotedCase) -> datetime | None: diff --git a/src/evalshift_cli/cli/commands/_suites.py b/src/evalshift_cli/cli/commands/_suites.py index cdf19a0..fe9c540 100644 --- a/src/evalshift_cli/cli/commands/_suites.py +++ b/src/evalshift_cli/cli/commands/_suites.py @@ -423,6 +423,39 @@ def format_rich(self) -> RenderableType: ) +def selected_suite_name( + *, + suite_path: Path | None, + suite_name: str | None, + cfg: EvalShiftConfig, +) -> str | None: + """The ``suites:`` key :func:`resolve_suite_path` resolves to, or ``None``. + + An explicit ``--suite`` path names no suite. ``--suite-name`` is returned as given (not + checked; :func:`resolve_suite_path` raises for an unknown one). With neither flag, the + single wired suite is auto-selected -- the bare ``evalshift compare`` that follows + ``capture sync``. Several wired suites, or none, select nothing. + + Args: + suite_path: Explicit ``--suite`` path, or ``None``. + suite_name: ``--suite-name`` key into ``cfg.suites``, or ``None``. + cfg: The loaded configuration (provides the ``suites`` mapping). + + Returns: + The selected suite's name, or ``None`` when the run is not on a named suite. + """ + if suite_path is not None: + return None + if suite_name is not None: + return suite_name + # Neither --suite nor --suite-name: prefer the wired suites over the + # bare-file default so the capture-first flow needs no extra flag. + if len(cfg.suites) == 1: + (only,) = cfg.suites + return only + return None + + def resolve_suite_path( *, suite_path: Path | None, @@ -456,20 +489,16 @@ def resolve_suite_path( config_dir = config_path.resolve().parent if suite_path is not None: return suite_path - if suite_name is not None: - entry = cfg.suites.get(suite_name) + name = selected_suite_name(suite_path=None, suite_name=suite_name, cfg=cfg) + if name is not None: + entry = cfg.suites.get(name) if entry is None: raise UnknownSuiteNameError( - name=suite_name, + name=name, known=sorted(cfg.suites), invocation=format_invocation(), ) return config_dir / entry.path - # Neither --suite nor --suite-name: prefer the wired suites over the - # bare-file default so the capture-first flow needs no extra flag. - if len(cfg.suites) == 1: - (entry,) = cfg.suites.values() - return config_dir / entry.path if len(cfg.suites) > 1: raise AmbiguousSuiteError( suite_names=sorted(cfg.suites), @@ -532,5 +561,6 @@ def resolve_suite_override( "render_suites_yaml", "resolve_suite_override", "resolve_suite_path", + "selected_suite_name", "suite_entry_payload", ] diff --git a/src/evalshift_cli/cli/commands/compare.py b/src/evalshift_cli/cli/commands/compare.py index a4edfeb..9cb2e5f 100644 --- a/src/evalshift_cli/cli/commands/compare.py +++ b/src/evalshift_cli/cli/commands/compare.py @@ -43,6 +43,7 @@ UnknownSuiteNameError, derive_suite_slug, resolve_suite_path, + selected_suite_name, ) from evalshift_cli.cli.commands.analyze import ( GATE_SEVERITIES, @@ -355,18 +356,22 @@ def _warn_if_legacy_name(ctx: typer.Context) -> None: def _freshness_lines( - cfg: EvalShiftConfig, suite_name: str | None, suite_path: Path, source_model: str + cfg: EvalShiftConfig, suite_name: str | None, suite_path: Path, source_model: str | None ) -> list[str]: """``⚠`` lines for a stale captured suite (spec §7). - compare drops doctor's ``warn`` rows, so it asks for the findings itself. Only for a suite - selected by name whose entry is ``source: captured``, and only with a capture limit set. + compare drops doctor's ``warn`` rows, so it asks for the findings itself. Only for a named + suite (``--suite-name`` or the single wired suite auto-selected) whose entry is + ``source: captured``, and only with a capture limit set. Args: cfg: The loaded configuration. - suite_name: The ``--suite-name`` selected, or ``None`` for a suite given by path. + suite_name: The selected suite's name (see + :func:`~evalshift_cli.cli.commands._suites.selected_suite_name`), or ``None`` for + a suite given by path. suite_path: The resolved ``golden.jsonl``; its case files sit beside it. - source_model: The source model this compare runs, which cases should be recorded on. + source_model: ``defaults.source_model``, which cases should be recorded on; ``None`` + turns the model findings off. Returns: One Rich-markup line per finding; empty when there is nothing to say. @@ -521,6 +526,8 @@ def compare_command( ) raise typer.Exit(code=1) + # Before ``suite_path`` is rebound to the resolved file: the explicit --suite decides it. + selected_suite = selected_suite_name(suite_path=suite_path, suite_name=suite_name, cfg=cfg) try: suite_path = resolve_suite_path( suite_path=suite_path, @@ -537,7 +544,9 @@ def compare_command( except SuiteError as exc: console.print(exc.format_rich()) raise typer.Exit(code=1) from exc - freshness = _freshness_lines(cfg, suite_name, suite_path, source_model) + # Staleness is measured against defaults.source_model (spec §3), as doctor and sync do; + # --from only picks what this run calls. + freshness = _freshness_lines(cfg, selected_suite, suite_path, cfg.defaults.source_model) run_slug = derive_suite_slug(suite_name=suite_name, suite_path=suite_path) diff --git a/src/evalshift_cli/cli/commands/doctor.py b/src/evalshift_cli/cli/commands/doctor.py index 8d2ed19..2dbef51 100644 --- a/src/evalshift_cli/cli/commands/doctor.py +++ b/src/evalshift_cli/cli/commands/doctor.py @@ -301,9 +301,10 @@ def _captures_store_check(cwd: Path) -> list[CheckResult]: def _suite_freshness_checks(cwd: Path) -> list[CheckResult]: """Bounded-suite rows (spec §7). Never ``fail``: freshness is advice. - ``captures limits`` warns when ``captures.store`` is set without ``max_age`` / - ``max_cases`` -- every sync then adds to the suite. With a limit set, each - ``source: captured`` suite gets one ``ok`` row, or one ``warn`` row per finding. + ``captures limits`` warns when ``captures.store`` is set without ``max_cases`` -- every + sync then adds to the suite; ``max_age`` alone bounds the fetch, not the suite (spec §5). + With either limit set, each ``source: captured`` suite also gets one ``ok`` row, or one + ``warn`` row per finding. """ cfg_path = cwd / CONFIG_FILENAME if not cfg_path.exists(): @@ -313,19 +314,21 @@ def _suite_freshness_checks(cwd: Path) -> list[CheckResult]: except ConfigError: return [] captures = cfg.captures - if captures.max_age is None and captures.max_cases is None: - if captures.store is None: - return [] - return [ + rows: list[CheckResult] = [] + if captures.store is not None and captures.max_cases is None: + missing = "captures.max_cases" + ( + " and captures.max_age" if captures.max_age is None else "" + ) + rows.append( CheckResult( name=CAPTURES_LIMITS_CHECK, status="warn", - detail="suites grow with every capture sync; " - "set captures.max_cases and captures.max_age", + detail=f"suites grow with every capture sync; set {missing}", ) - ] + ) + if captures.max_age is None and captures.max_cases is None: + return rows now = datetime.now(UTC) - rows: list[CheckResult] = [] for name, path in _named_suite_paths(cwd, cfg): entry = cfg.suites.get(name) if entry is None or entry.source != "captured": diff --git a/tests/unit/test_captures_staleness.py b/tests/unit/test_captures_staleness.py index 51540fb..3272b04 100644 --- a/tests/unit/test_captures_staleness.py +++ b/tests/unit/test_captures_staleness.py @@ -54,7 +54,9 @@ def test_parse_instant_normalises_to_utc(text: str, expected: datetime) -> None: assert parse_instant(text) == expected -@pytest.mark.parametrize("text", ["", " ", "yesterday", "2026-13-01"]) +@pytest.mark.parametrize( + "text", ["", " ", "yesterday", "2026-13-01", "0001-01-01T00:00:00+05:00"] +) def test_parse_instant_is_none_for_anything_unparseable(text: str) -> None: assert parse_instant(text) is None diff --git a/tests/unit/test_compare_command.py b/tests/unit/test_compare_command.py index 44321c6..d23a691 100644 --- a/tests/unit/test_compare_command.py +++ b/tests/unit/test_compare_command.py @@ -268,13 +268,14 @@ async def fake_complete(self: ModelClient, **kwargs: Any) -> CompletionResult: """A realistic hosted run URL: 102 characters, well past the 80-column fold.""" -def _captured_cases(tmp_path: Path, suite: str, n: int) -> Path: +def _captured_cases(tmp_path: Path, suite: str, n: int, *, model: str = "") -> Path: for i in range(n): write_promoted_case( PromotedCase( name=f"ex{i}", suite=suite, from_capture=f"cap_{i}", + source_model=model, example=SuiteExample(id=f"ex{i}", inputs={"name": f"User{i}"}, tools=[]), ), base=tmp_path / ".evalshift", @@ -474,11 +475,11 @@ def test_all_prints_the_tool_free_warning_it_scored_quietly( ) in flat assert flat.count("checked nothing") == 1 - def test_compare_warns_when_the_suite_is_over_its_cap( - self, - monkeypatch: pytest.MonkeyPatch, - tmp_path: Path, + @staticmethod + def _over_cap_project( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, model: str = "" ) -> None: + """One wired captured suite ``greet``: 3 cases against ``captures.max_cases: 2``.""" _scaffold(tmp_path) config_path = tmp_path / "evalshift.yaml" config_path.write_text( @@ -489,10 +490,17 @@ def test_compare_warns_when_the_suite_is_over_its_cap( + " path: .evalshift/suites/greet/golden.jsonl\n", encoding="utf-8", ) - _captured_cases(tmp_path, "greet", 3) + _captured_cases(tmp_path, "greet", 3, model=model) _patch_client(monkeypatch) monkeypatch.chdir(tmp_path) + def test_compare_warns_when_the_suite_is_over_its_cap( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + self._over_cap_project(tmp_path, monkeypatch) + result = runner.invoke(app, ["compare", "--suite-name", "greet", "--yes"]) assert result.exit_code == 0, result.output @@ -500,6 +508,49 @@ def test_compare_warns_when_the_suite_is_over_its_cap( assert "greet: 3 cases, over captures.max_cases 2; evalshift capture sync trims it" in flat assert flat.count("over captures.max_cases") == 1 + def test_compare_warns_for_the_auto_selected_suite( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """Bare ``compare`` -- the capture-first CI flow -- picks the one wired suite.""" + self._over_cap_project(tmp_path, monkeypatch) + + result = runner.invoke(app, ["compare", "--yes"]) + + assert result.exit_code == 0, result.output + flat = " ".join(result.output.split()) + assert "greet: 3 cases, over captures.max_cases 2; evalshift capture sync trims it" in flat + assert flat.count("over captures.max_cases") == 1 + + def test_freshness_measures_against_defaults_source_model_not_from( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ) -> None: + """``--from`` picks what to run; staleness is defined by ``defaults.source_model``.""" + self._over_cap_project(tmp_path, monkeypatch, model="gemini-2.5-flash") + + result = runner.invoke( + app, + [ + "compare", + "--suite-name", + "greet", + "--from", + "gemini-2.5-pro", + "--to", + "gemini-2.5-flash", + "--yes", + ], + ) + + assert result.exit_code == 0, result.output + flat = " ".join(result.output.split()) + assert flat.count("over captures.max_cases") == 1 + assert "no case recorded on" not in flat + assert "recorded on another model" not in flat + def test_all_pushes_before_gate_exit( self, monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_doctor.py b/tests/unit/test_doctor.py index 0234f5e..faf1f14 100644 --- a/tests/unit/test_doctor.py +++ b/tests/unit/test_doctor.py @@ -987,3 +987,36 @@ def test_a_bucket_without_limits_gets_a_nudge( assert row.detail == ( "suites grow with every capture sync; set captures.max_cases and captures.max_age" ) + + def test_a_bucket_with_max_age_only_still_grows( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + # max_age bounds what a sync fetches, not how many cases the suite keeps (spec §5). + from tests.unit.fake_remote_store import populated_store + + self._project( + tmp_path, + captures="captures:\n store: s3://acme-evals/p\n max_age: 14d\n", + ages_days=[23], + ) + monkeypatch.setattr(doctor_module, "open_store", lambda parsed: populated_store("cap_1")) + [limits] = self._rows(tmp_path, CAPTURES_LIMITS_CHECK) + assert limits.status == "warn" + assert limits.detail == "suites grow with every capture sync; set captures.max_cases" + [fresh] = self._rows(tmp_path) + assert fresh.status == "warn" + assert "newest case captured 23d ago (captures.max_age 14d)" in fresh.detail + + def test_a_bucket_with_max_cases_needs_no_nudge( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + from tests.unit.fake_remote_store import populated_store + + self._project( + tmp_path, + captures="captures:\n store: s3://acme-evals/p\n max_cases: 5\n", + ages_days=[1], + ) + monkeypatch.setattr(doctor_module, "open_store", lambda parsed: populated_store("cap_1")) + assert self._rows(tmp_path, CAPTURES_LIMITS_CHECK) == [] + assert [r.status for r in self._rows(tmp_path)] == ["ok"] diff --git a/tests/unit/test_suites_helpers.py b/tests/unit/test_suites_helpers.py index 90ff629..93bec4a 100644 --- a/tests/unit/test_suites_helpers.py +++ b/tests/unit/test_suites_helpers.py @@ -21,6 +21,7 @@ render_suites_region, render_suites_yaml, resolve_suite_path, + selected_suite_name, suite_entry_payload, ) from evalshift_cli.config.models import EvalShiftConfig @@ -84,6 +85,31 @@ def test_no_suites_falls_back_to_golden_file(self) -> None: assert got == Path(SUITE_FILENAME) +class TestSelectedSuiteName: + """The ``suites:`` key :func:`resolve_suite_path` resolves to, if any.""" + + def _name(self, cfg: EvalShiftConfig, **kw: Any) -> str | None: + return selected_suite_name( + suite_path=kw.get("suite_path"), suite_name=kw.get("suite_name"), cfg=cfg + ) + + def test_explicit_suite_path_names_no_suite(self) -> None: + cfg = _config(suites={"only": {"path": "a.jsonl"}}) + assert self._name(cfg, suite_path=Path("a.jsonl"), suite_name="only") is None + + def test_suite_name_is_returned_as_given(self) -> None: + cfg = _config(suites={"a": {"path": "a.jsonl"}, "b": {"path": "b.jsonl"}}) + assert self._name(cfg, suite_name="b") == "b" + + def test_single_wired_suite_is_auto_selected(self) -> None: + cfg = _config(suites={"only": {"path": "a.jsonl"}}) + assert self._name(cfg) == "only" + + def test_several_or_no_wired_suites_select_nothing(self) -> None: + assert self._name(_config()) is None + assert self._name(_config(suites={"a": {"path": "a"}, "b": {"path": "b"}})) is None + + class TestDeriveSuiteSlug: def test_prefers_explicit_suite_name(self) -> None: slug = derive_suite_slug( From f872a742aa1188ac27a1f1fdf7b745daa79a5f4a Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 23:35:47 +0200 Subject: [PATCH 16/26] test(capture): weeks of bucket captures stay one bounded fresh suite Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_capture_lifecycle.py | 144 ++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 tests/integration/test_capture_lifecycle.py diff --git a/tests/integration/test_capture_lifecycle.py b/tests/integration/test_capture_lifecycle.py new file mode 100644 index 0000000..1715ac7 --- /dev/null +++ b/tests/integration/test_capture_lifecycle.py @@ -0,0 +1,144 @@ +"""Weeks of production captures in a bucket stay one bounded, fresh suite (spec, Testing).""" + +from __future__ import annotations + +import re +import shutil +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import pytest +from typer.testing import CliRunner + +from evalshift_cli.captures.store_uri import StoreURI +from evalshift_cli.cli.commands import capture as capture_module +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config +from evalshift_cli.cli.main import app +from tests.unit.fake_remote_store import ( + TOOLSET_KEY, + TOOLSET_PAYLOAD, + FakeRemoteStore, + capture_payload, +) + +runner = CliRunner() + + +def _day(day: int, month: int = 6) -> datetime: + return datetime(2026, month, day, 12, tzinfo=UTC) + + +class _Bucket: + """A fake bucket the SDK 'writes' to between syncs.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.store = FakeRemoteStore({TOOLSET_KEY: TOOLSET_PAYLOAD}) + monkeypatch.setattr(capture_module, "open_store", lambda parsed: self._open(parsed)) + + def _open(self, parsed: StoreURI) -> FakeRemoteStore: + return self.store + + def record(self, cid: str, when: datetime, *, model: str = "claude-sonnet-5-20260301") -> None: + key = f"captures/planner/{cid}.json" + self.store.objects[key] = capture_payload( + cid, + "planner", + model_input=f"plan my day {cid}", + created_at=when.isoformat(), + model_id=model, + ) + self.store.modified[key] = when + + +def _config(path: Path, *, source_model: str = "claude-sonnet-5") -> None: + text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") + text = re.sub(r"(?m)^ source_model: .*$", f" source_model: {source_model}", text) + text += "\ncaptures:\n store: s3://acme-evals/testapp\n max_age: 14d\n max_cases: 4\n" + path.write_text(text, encoding="utf-8") + + +def _sync(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, now: datetime) -> Any: + monkeypatch.setattr(capture_module, "_now", lambda: now) + result = runner.invoke( + app, + ["capture", "sync", "--config", str(tmp_path / "evalshift.yaml"), "--base", str(tmp_path)], + ) + assert result.exit_code == 0, result.stdout + return result + + +def _cases(tmp_path: Path) -> set[str]: + return {p.stem for p in (tmp_path / "suites" / "planner").glob("*.json")} + + +def test_weeks_of_captures_stay_one_bounded_fresh_suite( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml") + + # Week 1: six captures, a suite of four. + for i in range(6): + bucket.record(f"w1_{i}", _day(10)) + _sync(tmp_path, monkeypatch, _day(12)) + week_1 = _cases(tmp_path) + assert len(week_1) == 4 + + # Week 2, inside max_age: more captures, but nothing is stale -- the suite holds still. + for i in range(6): + bucket.record(f"w2_{i}", _day(17)) + _sync(tmp_path, monkeypatch, _day(19)) + assert _cases(tmp_path) == week_1 + + # Two weeks later: week-1 cases are stale and fresh captures replace them. + for i in range(3): + bucket.record(f"w4_{i}", _day(1, month=7)) + _sync(tmp_path, monkeypatch, _day(2, month=7)) + cases = _cases(tmp_path) + assert len(cases) == 4 + assert {f"w4_{i}" for i in range(3)} <= cases + + # A fresh checkout: suites committed, local mirror gone. Promoted captures are not + # downloaded again, and nothing outside max_age is. + shutil.rmtree(tmp_path / "captures") + bucket.store.gets.clear() + result = _sync(tmp_path, monkeypatch, _day(2, month=7)) + downloaded = {key.rsplit("/", 1)[-1].removesuffix(".json") for key in bucket.store.gets} + assert not downloaded & cases + assert not any(cid.startswith(("w1_", "w2_")) for cid in downloaded) + assert _cases(tmp_path) == cases + assert "promoted" in result.stdout + + +def test_a_production_migration_retires_the_old_models_cases( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml") + for i in range(4): + bucket.record(f"old_{i}", _day(10)) + _sync(tmp_path, monkeypatch, _day(12)) + + _config(tmp_path / "evalshift.yaml", source_model="claude-sonnet-5-5") + for i in range(2): + bucket.record(f"new_{i}", _day(13), model="claude-sonnet-5-5-20260601") + result = _sync(tmp_path, monkeypatch, _day(14)) + + assert {"new_0", "new_1"} <= _cases(tmp_path) + assert len(_cases(tmp_path)) == 4 + assert "retired 2 (2 on another model)" in " ".join(result.stdout.split()) + + +def test_a_wrong_source_model_trips_the_guardrail( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml", source_model="gpt-9") + for i in range(3): + bucket.record(f"cap_{i}", _day(10)) + + result = _sync(tmp_path, monkeypatch, _day(12)) + + assert len(_cases(tmp_path)) == 3 + assert "not filtering by model — check defaults.source_model" in " ".join(result.stdout.split()) From fca89a81ad575e9fb28b9f24bbe83cd225aa1d38 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 23:43:46 +0200 Subject: [PATCH 17/26] test(capture): pin the fresh-checkout fetch and the guardrail on an established suite Co-Authored-By: Claude Opus 5.5 --- tests/integration/test_capture_lifecycle.py | 64 +++++++++++++++++---- 1 file changed, 53 insertions(+), 11 deletions(-) diff --git a/tests/integration/test_capture_lifecycle.py b/tests/integration/test_capture_lifecycle.py index 1715ac7..1de1251 100644 --- a/tests/integration/test_capture_lifecycle.py +++ b/tests/integration/test_capture_lifecycle.py @@ -22,6 +22,8 @@ capture_payload, ) +pytestmark = pytest.mark.integration + runner = CliRunner() @@ -52,8 +54,11 @@ def record(self, cid: str, when: datetime, *, model: str = "claude-sonnet-5-2026 def _config(path: Path, *, source_model: str = "claude-sonnet-5") -> None: - text = render_minimal_config(profile="model-upgrade").replace(CAPTURES_SCAFFOLD + "\n", "") - text = re.sub(r"(?m)^ source_model: .*$", f" source_model: {source_model}", text) + text = render_minimal_config(profile="model-upgrade") + assert CAPTURES_SCAFFOLD + "\n" in text + text = text.replace(CAPTURES_SCAFFOLD + "\n", "") + text, n = re.subn(r"(?m)^ source_model: .*$", f" source_model: {source_model}", text) + assert n == 1 text += "\ncaptures:\n store: s3://acme-evals/testapp\n max_age: 14d\n max_cases: 4\n" path.write_text(text, encoding="utf-8") @@ -68,6 +73,11 @@ def _sync(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, now: datetime) -> Any return result +def _said(result: Any) -> str: + """The command's stdout with Rich's line wrapping undone.""" + return " ".join(result.stdout.split()) + + def _cases(tmp_path: Path) -> set[str]: return {p.stem for p in (tmp_path / "suites" / "planner").glob("*.json")} @@ -88,8 +98,10 @@ def test_weeks_of_captures_stay_one_bounded_fresh_suite( # Week 2, inside max_age: more captures, but nothing is stale -- the suite holds still. for i in range(6): bucket.record(f"w2_{i}", _day(17)) - _sync(tmp_path, monkeypatch, _day(19)) + result = _sync(tmp_path, monkeypatch, _day(19)) assert _cases(tmp_path) == week_1 + # 8 eligible = six week-2 captures + the two week-1 captures the cap left out. + assert "planner: 4 cases — kept 4, retired 0, added 0 of 8 eligible" in _said(result) # Two weeks later: week-1 cases are stale and fresh captures replace them. for i in range(3): @@ -98,17 +110,21 @@ def test_weeks_of_captures_stay_one_bounded_fresh_suite( cases = _cases(tmp_path) assert len(cases) == 4 assert {f"w4_{i}" for i in range(3)} <= cases + # Only three replacements exist, so one stale week-1 case survives (replacement-only). + assert len(cases & week_1) == 1 # A fresh checkout: suites committed, local mirror gone. Promoted captures are not - # downloaded again, and nothing outside max_age is. + # downloaded again, and nothing outside max_age is -- only the one new capture is. shutil.rmtree(tmp_path / "captures") bucket.store.gets.clear() + bucket.record("w5_0", _day(2, month=7)) result = _sync(tmp_path, monkeypatch, _day(2, month=7)) - downloaded = {key.rsplit("/", 1)[-1].removesuffix(".json") for key in bucket.store.gets} - assert not downloaded & cases - assert not any(cid.startswith(("w1_", "w2_")) for cid in downloaded) - assert _cases(tmp_path) == cases - assert "promoted" in result.stdout + assert [k for k in bucket.store.gets if k.startswith("captures/")] == [ + "captures/planner/w5_0.json" + ] + assert "(skipped 4 promoted, 11 older than captures.max_age (14d))" in _said(result) + # The new capture takes the stale week-1 survivor's slot. + assert _cases(tmp_path) == {"w4_0", "w4_1", "w4_2", "w5_0"} def test_a_production_migration_retires_the_old_models_cases( @@ -127,7 +143,7 @@ def test_a_production_migration_retires_the_old_models_cases( assert {"new_0", "new_1"} <= _cases(tmp_path) assert len(_cases(tmp_path)) == 4 - assert "retired 2 (2 on another model)" in " ".join(result.stdout.split()) + assert "retired 2 (2 on another model)" in _said(result) def test_a_wrong_source_model_trips_the_guardrail( @@ -141,4 +157,30 @@ def test_a_wrong_source_model_trips_the_guardrail( result = _sync(tmp_path, monkeypatch, _day(12)) assert len(_cases(tmp_path)) == 3 - assert "not filtering by model — check defaults.source_model" in " ".join(result.stdout.split()) + assert "not filtering by model — check defaults.source_model" in _said(result) + + +def test_a_source_model_typo_neither_empties_nor_retires_an_established_suite( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + bucket = _Bucket(monkeypatch) + _config(tmp_path / "evalshift.yaml") + for i in range(4): + bucket.record(f"est_{i}", _day(10)) + _sync(tmp_path, monkeypatch, _day(12)) + established = _cases(tmp_path) + assert len(established) == 4 + + _config(tmp_path / "evalshift.yaml", source_model="claude-sonet-5") + for i in range(2): + bucket.record(f"more_{i}", _day(13)) + result = _sync(tmp_path, monkeypatch, _day(14)) + + # The guardrail switches the model check off, so all four cases stay fresh: zero free slots + # under max_cases 4, nothing stale to replace -- the suite holds, nothing on "another model". + said = _said(result) + assert "none of 6 case(s) and capture(s) were recorded on claude-sonet-5" in said + assert "not filtering by model — check defaults.source_model" in said + assert "planner: 4 cases — kept 4, retired 0, added 0 of 2 eligible" in said + assert "on another model" not in said + assert _cases(tmp_path) == established From c24121ab8c950a16e8452aa27f8a36bdb028ce08 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Fri, 16 Oct 2026 23:54:09 +0200 Subject: [PATCH 18/26] docs: bounded capture suites (captures.max_age / captures.max_cases) Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 60 ++++++ DOCS.md | 33 ++- docs/configuration.md | 189 +++++++++++++++++- docs/conversations.md | 4 + docs/sdk.md | 23 ++- .../2026-10-16-bounded-capture-suites.md | 15 ++ ...026-10-16-bounded-capture-suites-design.md | 12 +- llms-full.txt | 116 ++++++++++- 8 files changed, 424 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e3ecfc..ea68077 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,66 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- **Bounded capture suites.** Two optional settings under `captures:`, + `max_age: 14d` and `max_cases: 200`, keep each captured suite a capped, + route-balanced sample of recent production on `defaults.source_model`. + Setting either one turns the feature on. `capture sync` then promotes only + captures recorded within `max_age` on the source model. With `max_cases` + set, it never grows a suite past the cap and swaps stale cases (too old, or + recorded on another model) for fresh captures as they arrive. A stale case + whose question a fresh capture asks again is retired at once. Any other + stale case leaves only to make room, so a suite that has nothing to replace + it with never shrinks. A conversation enters whole or not at all. Cases a + human marked `provenance: reviewed` are never retired. The same captures + give the same suite on any machine, and syncing again with nothing new + changes no file. `max_cases` also trims suites sync did not build: suites + promoted by hand with `capture promote`, and suites whose entry is + `managed: false` (that flag freezes the YAML entry, not the case files). + Model ids match on bare names: lower-cased, provider prefix and trailing + snapshot date dropped, no alias lookup. If none of a suite's cases and + captures was recorded on `defaults.source_model`, sync warns + (`none of N case(s) and capture(s) were recorded on …; not filtering + by model — check defaults.source_model`) and does not filter that suite by + model rather than empty it. `max_age` also bounds what `capture sync`'s + automatic fetch downloads from `captures.store`; `--since` still overrides + it, and `capture list` and `capture fetch` are unchanged. Sync prints one + line per suite: `alpha: 200 cases — kept 150, retired 50 (32 older than + 14d, 18 on another model), added 50 of 2940 eligible; not eligible: …`. + A suite whose case files cannot be read is left untouched, with a warning, + and the other suites still sync. +- `doctor` gains `suite freshness` rows for each `source: captured` suite when + either limit is set: `ok` (`alpha: 200 cases, captured 2d–12d ago`), or one + `warn` per finding — the newest case is older than `max_age`, cases were + recorded on another model than `defaults.source_model` (or none on it), the + suite is over `max_cases`, or case files could not be read. A + `captures limits` warning fires when `captures.store` is set and + `captures.max_cases` is not (`suites grow with every capture sync; set + captures.max_cases and captures.max_age`). `compare` prints the same + findings above the verdict for the suite it runs, named with `--suite-name` + or auto-selected (not for `--suite `), so a team that only runs CI + sees its suite went stale. +- Promoted case files record `captured_at` and `source_model`, with or without + the limits. `capture sync` backfills them on existing cases whose capture is + still local (`backfilled capture time and model on N case(s)`). + +### Changed + +- `evalshift init` scaffolds `captures: {max_age: 14d, max_cases: 200}` (with a + commented `store:`). Existing configs are untouched. Without the two keys, + `capture sync` keeps and promotes exactly what it did before. +- `capture sync` now reads `evalshift.yaml` under `--offline` too, for the + limits. When the file cannot be read and has a top-level `captures:` key, + sync says so (`could not read (); syncing without + captures.max_age / captures.max_cases.`), online and offline. +- Older CLIs reject the new `captures` keys and the new case-file fields: bump + the CLI everywhere `capture` commands run (and a pinned CI action's + `evalshift-version`) before adding them. CI's `compare` reads only + `golden.jsonl`, whose format is unchanged. A run left in progress by an + older CLI cannot be `--resume`d across the upgrade (config-hash mismatch); + start a fresh one. + ## [1.5.0] - 2026-10-15 ### Added diff --git a/DOCS.md b/DOCS.md index 4222090..65d2439 100644 --- a/DOCS.md +++ b/DOCS.md @@ -130,7 +130,7 @@ See [Project setup](#project-setup) and [Capturing from production](#capturing-f ## Project setup -`evalshift init` is the real-project entry point. Writes **only** a minimal, capture-first `evalshift.yaml`: a passthrough `replay` prompt (`content: "{input}"`), advisory LLM-judge and semantic evaluators (both active for every provider), an empty managed `suites:` block for `capture sync` to fill, and a migration policy. The intended flow: instrument your agent with the evalshift-sdk → record captures → `evalshift capture sync` → run against the promoted suite. +`evalshift init` is the real-project entry point. Writes **only** a minimal, capture-first `evalshift.yaml`: a passthrough `replay` prompt (`content: "{input}"`), advisory LLM-judge and semantic evaluators (both active for every provider), a migration policy, `captures:` limits (`max_age: 14d`, `max_cases: 200`, with `store:` commented out; see [Bounded suites](#capturing-from-production)), and an empty managed `suites:` block for `capture sync` to fill. The intended flow: instrument your agent with the evalshift-sdk → record captures → `evalshift capture sync` → run against the promoted suite. `init` options: @@ -171,10 +171,14 @@ evalshift capture clean # delete promoted capture files + sweep o ```yaml captures: store: s3://acme-evals/support-agent + max_age: 14d # optional: see Bounded suites below + max_cases: 200 # optional, per suite ``` and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download; `capture sync --repair` alone re-fetches the promoted captures its rebuilds need, past `--since` too), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching client library — `pip install boto3`, `google-cloud-storage` or `azure-storage-blob azure-identity` — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, at most its scheme. A missing client library exits 1 naming the `pip install ` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). +**Bounded suites.** `captures.max_age` (a duration: `30m`, `24h`, `14d`) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) whose question a new capture asks again is retired outright; any other stale case leaves only to make room, other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). + **`capture sync`** is the workhorse. Per suite it: 1. Groups captures (by `conversation_id` for multi-turn, ordered by `turn_index`). @@ -185,8 +189,9 @@ and `capture sync` and `capture list` fetch what is new before they read the loc 6. Skips captures whose replayed content duplicates an already-promoted case or an earlier capture in the same run — duplicate examples inflate *n* and corrupt the paired statistics (`--keep-duplicates` opts out). Dedup is seeded from the cases already in the suite dir, so it holds across repeated syncs. 7. Skips captures whose turn recorded an `error` event — a turn that died before the agent acted is not ground truth, and promoting it would assert `expected_no_tools: true` on a question that needed a tool. `--allow-errored` promotes it anyway (still never asserting `expected_no_tools`). `capture promote` exits non-zero on the same condition. Separately and unconditionally — `--allow-errored` does not help — a capture whose first `model_call` has no `toolset_ref` is refused: the SDK did not record what tools were offered, so there is nothing to carry, and re-capturing with a current `evalshift-sdk` is the only fix. Also unconditionally, a capture whose recorded messages carry content a text replay can't carry — an image, a document, audio, a server-tool result, an unknown block type — is skipped and counted on its own (`capture promote` refuses it): a replay without it is not ground truth. Last, as a safety net, a capture whose case fails the suite schema's own validation in a way none of these rules anticipates is skipped with pydantic's first error and counted on its own (`skipped N capture(s) that could not form a valid case`; `capture promote` refuses it) instead of aborting the sync; hand-write that case or report the capture. 8. Warns when two captures claim the same `(conversation_id, turn_index)` (a retried turn), and when a promoted turn contains a failed tool result (`error`, or `{"success": false}`). Both stay warnings — see [Agent evals → What does not belong in a golden suite](docs/agents.md). -9. Writes `.evalshift/suites//golden.jsonl` and rewrites the managed `suites:` block in `evalshift.yaml` (between the `>>> evalshift suites` markers). -10. After the write (or after printing the block for you to paste), checks the CI pin: if a workflow under `.github/workflows/` uses `evalshift/evalshift-action` with an `evalshift-version` older than this CLI, with no pin at all, or with pins that are all newer than this CLI, it prints a warning naming the workflow and job plus the fix — the exact `evalshift-version: ""` line to set, or, for a newer pin, `pip install -U evalshift` locally. Advisory only — sync never edits a workflow and the exit code is unchanged. See [Pin drift](#pin-drift). +9. With `captures.max_age` / `captures.max_cases` set: filters, plans, retires. Captures recorded before now − `max_age`, or on another model than `defaults.source_model`, are not eligible. With `max_cases` set, sync plans each suite (every suite directory with case files, not only those with new captures; `--suite` narrows it). It fills the free slots route-balanced, a conversation at a time. It deletes the case files of the stale cases it retires and of any fresh cases it trims, and rebuilds `golden.jsonl` from what is left; reviewed cases stay. A suite whose case files cannot be read is left unchanged, with a warning (`alpha: could not read its promoted cases (ValidationError); left it unchanged.`). After the summary line, sync prints one line per suite: ` planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model` (zero counts left out; `max_age` alone prints only ` planner: not eligible: …`). Syncing again with nothing new changes no file. If `evalshift.yaml` cannot be read, sync runs without the limits and, when the file has a top-level `captures:` key, says so: `could not read evalshift.yaml (); syncing without captures.max_age / captures.max_cases.` Sync reads the file under `--offline` too. +10. Writes `.evalshift/suites//golden.jsonl` and rewrites the managed `suites:` block in `evalshift.yaml` (between the `>>> evalshift suites` markers). +11. After the write (or after printing the block for you to paste), checks the CI pin: if a workflow under `.github/workflows/` uses `evalshift/evalshift-action` with an `evalshift-version` older than this CLI, with no pin at all, or with pins that are all newer than this CLI, it prints a warning naming the workflow and job plus the fix — the exact `evalshift-version: ""` line to set, or, for a newer pin, `pip install -U evalshift` locally. Advisory only — sync never edits a workflow and the exit code is unchanged. See [Pin drift](#pin-drift). Strictness knobs for the derived tool expectations: `--strict-args` (exact argument matches), `--names-only` (ignore arguments), `--tool-count` (also pin the call count, scoped the same way as `expected_tools`), `--rounds {first,all}` (which agent rounds become ground truth, default `first`). `--tag` attaches extra slice tags; `--print` previews the `suites:` block without writing. @@ -217,12 +222,12 @@ init → doctor → run → evaluate → analy .json (if policy) ``` -- **`doctor`** validates local config and shows which provider keys are visible. Exit 1 only when an existing `evalshift.yaml` fails validation — the row gives the problem count and points at `evalshift validate`, which prints each problem — or when a blocking `semantic` / `llm_judge` model has no API key; other missing keys are soft warnings. Its second row, `evalshift-sdk`, reports the SDK version the `evalshift` import name resolves to in this environment (`warn` when the SDK is missing or shadowed by an older CLI's leftover files; never a failure). It also reports the toolset each configured suite carries (or the flat `golden.jsonl`) and flags a suite whose examples carry more than one distinct toolset — legal (each example dispatches its own), but also the shape a wiring mistake takes. When a workflow under `.github/workflows/` uses the GitHub Action it adds a `ci pin` row: `ok` (`pinned to `) when CI installs this CLI version, `warn` when the pin is older, absent, or newer than the local CLI (see [Pin drift](#pin-drift)). When the config sets `captures.store`, a `captures.store` row reports it: `warn` naming the `pip install ` to run when the client library is missing (the capture commands that need it exit 1 themselves), `warn` when the bucket cannot be listed (no cloud credentials on this machine, say), `ok` (` reachable`) otherwise; no row without a store. When the config wires an `llm_judge` evaluator and names both `defaults.source_model` and `target_model`, a `judge family` row warns for every `judge_model` that resolves to the same provider as an arm (self-preference bias; `ok` "from a third family" otherwise, no row when either arm is unset) — advisory, never a failure; `validate` prints the same line and the report repeats it above the verdict (see [`evaluators.llm_judge`](docs/configuration.md#evaluatorsllm_judge)). When the config has a `semantic` or `llm_judge` evaluator, an `evaluator keys` row checks every judge and embedding model across the top-level block and every named suite's: `fail` (exit 1) for a blocking evaluator with no key, `warn` for an advisory one (skipped until the key is exported), one `ok` row when every such model has a key (see [Missing judge or embedding keys](#missing-judge-or-embedding-keys)). +- **`doctor`** validates local config and shows which provider keys are visible. Exit 1 only when an existing `evalshift.yaml` fails validation — the row gives the problem count and points at `evalshift validate`, which prints each problem — or when a blocking `semantic` / `llm_judge` model has no API key; other missing keys are soft warnings. Its second row, `evalshift-sdk`, reports the SDK version the `evalshift` import name resolves to in this environment (`warn` when the SDK is missing or shadowed by an older CLI's leftover files; never a failure). It also reports the toolset each configured suite carries (or the flat `golden.jsonl`) and flags a suite whose examples carry more than one distinct toolset — legal (each example dispatches its own), but also the shape a wiring mistake takes. When a workflow under `.github/workflows/` uses the GitHub Action it adds a `ci pin` row: `ok` (`pinned to `) when CI installs this CLI version, `warn` when the pin is older, absent, or newer than the local CLI (see [Pin drift](#pin-drift)). When the config sets `captures.store`, a `captures.store` row reports it: `warn` naming the `pip install ` to run when the client library is missing (the capture commands that need it exit 1 themselves), `warn` when the bucket cannot be listed (no cloud credentials on this machine, say), `ok` (` reachable`) otherwise; no row without a store. When `captures.store` is set and `captures.max_cases` is not, a `captures limits` row warns `suites grow with every capture sync; set captures.max_cases and captures.max_age` (`…; set captures.max_cases` when `max_age` is set). With either limit set, each `source: captured` suite gets a `suite freshness` row: `ok` (`alpha: 200 cases, captured 2d–12d ago`), or one `warn` per finding — newest case older than `max_age`, cases recorded on another model than `defaults.source_model` (or none on it), over `max_cases`, unreadable case files (see [Bounded suites](docs/configuration.md#freshness-in-doctor-and-compare)). When the config wires an `llm_judge` evaluator and names both `defaults.source_model` and `target_model`, a `judge family` row warns for every `judge_model` that resolves to the same provider as an arm (self-preference bias; `ok` "from a third family" otherwise, no row when either arm is unset) — advisory, never a failure; `validate` prints the same line and the report repeats it above the verdict (see [`evaluators.llm_judge`](docs/configuration.md#evaluatorsllm_judge)). When the config has a `semantic` or `llm_judge` evaluator, an `evaluator keys` row checks every judge and embedding model across the top-level block and every named suite's: `fail` (exit 1) for a blocking evaluator with no key, `warn` for an advisory one (skipped until the key is exported), one `ok` row when every such model has a key (see [Missing judge or embedding keys](#missing-judge-or-embedding-keys)). - **`run`** parses prompts, validates every example against every prompt, estimates cost, then dispatches `(prompt × example × {source, target})` calls through an async orchestrator under a concurrency semaphore. Responses are cached — per call, and per replayed round for tool-calling examples (see [Response cache](#response-cache)); progress is checkpointed every 50 completions. - **`evaluate`** scores each (source, target) pair with the configured evaluators, one `EvalRecord` per pair × evaluator. Scoring runs under the same `defaults.concurrency` semaphore as `run`, and the embedding/judge calls it makes go through the same response cache. - **`analyze`** runs paired statistics per `(prompt, evaluator, slice)`, applies Benjamini–Hochberg FDR correction, classifies severities, and — when a `migration_policy` is configured — computes a pass/fail verdict. - **`report`** renders the single-file HTML report (no external assets; works offline and attaches cleanly to a PR or email), and writes the machine-written [run insights](#run-insights) narrative unless `--no-insights` is passed. The page opens on a verdict / advisory-signal / economics panel row and a six-cell run strip (examples, calls, failed-or-truncated, spend, latency Δ, mean score Δ), then the executive summary (one row per prompt, drawn from blocking evaluators — advisory ones only on a prompt nothing else scored; the strip's mean score Δ averages those rows), the narrative, the **Trace rules broken** panel when the target broke a hand-written trace rule, one section per prompt, and the methodology. Every figure on it is derived from the run's own artefacts; the deltas in the header are the run-level rollup of the per-prompt economics. Top regressions are collapsed cards — expand one for the trace diff, the tool diffs and the conversation context. The report is dark-only. -- **`compare`** chains everything end to end over one suite and adds `--gate`, `--policy-gate`, `--push`, `--open`. Warnings raised while the pipeline runs (LiteLLM deprecation notices, insights-retry notes) are deferred and printed as one `⚠` section directly under the pipeline block — also when a stage fails, since a held-back warning may explain the failure. Errors are never deferred. +- **`compare`** chains everything end to end over one suite and adds `--gate`, `--policy-gate`, `--push`, `--open`. Warnings raised while the pipeline runs (LiteLLM deprecation notices, insights-retry notes) are deferred and printed as one `⚠` section directly under the pipeline block — also when a stage fails, since a held-back warning may explain the failure. Errors are never deferred. With `captures.max_age` or `captures.max_cases` set, `compare` prints doctor's `suite freshness` findings for the suite it runs (named with `--suite-name` or auto-selected; not for `--suite `) as `⚠` lines above the verdict, so a CI log shows a stale suite. ### Artefacts @@ -309,6 +314,16 @@ migration_policy: max_cost_increase: 0.30 max_latency_increase: 0.30 +# How captures become suites. Each suite stays a capped sample of production +# from the last max_age, recorded on defaults.source_model: `capture sync` +# swaps stale cases for fresh captures as they arrive. Does the SDK ship +# captures to a bucket (EVALSHIFT_CAPTURE_STORE on the agent)? Uncomment +# store: and name the same bucket. +captures: + max_age: 14d + max_cases: 200 + # store: s3:/// + # The managed region goes last: it is the only part of the file a command # rewrites, and the only part that grows without bound (one entry per suite, # each with its own evaluator block). @@ -329,7 +344,7 @@ suites: {} | `migration_policy` | block \| absent | Regression budgets, see [Migration policy](#migration-policy-and-ci-gating) | | `suites` | map | Named suites (`{name: {source: captured\|jsonl, path: ..., evaluators: ..., managed: true}}`); the block between the `>>> evalshift suites` markers is managed by `capture sync`. See [Per-suite evaluators](#per-suite-evaluators) | | `retention` | block | `max_runs_per_suite` (default 20, `0` disables), `run_ttl_days` (default off) | -| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. See [Capturing from production](#capturing-from-production) | +| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. `max_age` (duration, `30m`/`24h`/`14d`) and `max_cases` (int ≥ 1), both default unset: setting either keeps each captured suite a capped, route-balanced sample of recent production on `defaults.source_model` (bounded suites). See [Capturing from production](#capturing-from-production) | `thresholds` was a top-level field until it was removed. It was free-form and gated nothing — `migration_policy` is the single source of truth for gating — so rather than being silently ignored it is now rejected by name, and a config that still carries it fails to load: @@ -887,9 +902,9 @@ Common conventions: `-c/--config` defaults to `./evalshift.yaml`; run artefacts **`evalshift init`** — scaffold a minimal capture-first `evalshift.yaml`. `-f/--force` · `-d/--directory ` · `--ci` · `--wire-agents/--no-wire-agents` (default on) · `--provider gemini|openai|anthropic|deepseek` · `--profile model-upgrade|cost-reduction|local-model|quantization|provider-switch` (default `model-upgrade`) -Without `--ci`, warns after writing when an existing workflow under `.github/workflows/` pins an older or newer CLI than this one, or none at all (see [Pin drift](#pin-drift)); `init --ci` writes the pin itself and does not warn about the file it just wrote. +Without `--ci`, warns after writing when an existing workflow under `.github/workflows/` pins an older or newer CLI than this one, or none at all (see [Pin drift](#pin-drift)); `init --ci` writes the pin itself and does not warn about the file it just wrote. Scaffolds `captures: {max_age: 14d, max_cases: 200}` with `store:` commented out (see [Capturing from production](#capturing-from-production)). -**`evalshift doctor`** — environment/config check. Exit 1 only on an invalid existing config or a blocking `semantic` / `llm_judge` model with no API key. Row 2, `evalshift-sdk`, confirms `import evalshift` is the SDK (`warn` when missing or shadowed, never a failure). Reports the toolset each configured suite carries and flags a suite whose examples carry more than one distinct toolset. The suite-side checks cover every suite in the config's `suites:` block, falling back to `./golden.jsonl` when none are wired. Adds a `ci pin` row when a workflow uses the GitHub Action (`warn` on pin drift, never a failure) and a `judge family` row when an `llm_judge` judge shares a provider with a configured arm (`warn`, never a failure). Adds a `captures.store` row when the config names a store (`warn` naming the package to install when the client library is missing, `warn` when the bucket cannot be listed), and an `evaluator keys` row when the config has a `semantic` or `llm_judge` evaluator (`fail` for a blocking one whose model has no key, `warn` for an advisory one, `ok` otherwise; top-level block and every named suite). +**`evalshift doctor`** — environment/config check. Exit 1 only on an invalid existing config or a blocking `semantic` / `llm_judge` model with no API key. Row 2, `evalshift-sdk`, confirms `import evalshift` is the SDK (`warn` when missing or shadowed, never a failure). Reports the toolset each configured suite carries and flags a suite whose examples carry more than one distinct toolset. The suite-side checks cover every suite in the config's `suites:` block, falling back to `./golden.jsonl` when none are wired. Adds a `ci pin` row when a workflow uses the GitHub Action (`warn` on pin drift, never a failure) and a `judge family` row when an `llm_judge` judge shares a provider with a configured arm (`warn`, never a failure). Adds a `captures.store` row when the config names a store (`warn` naming the package to install when the client library is missing, `warn` when the bucket cannot be listed), and an `evaluator keys` row when the config has a `semantic` or `llm_judge` evaluator (`fail` for a blocking one whose model has no key, `warn` for an advisory one, `ok` otherwise; top-level block and every named suite). Adds a `captures limits` row (`warn`) when `captures.store` is set without `captures.max_cases`, and, with either capture limit set, a `suite freshness` row per `source: captured` suite (`ok`, or `warn` per finding; never a failure). **`evalshift run`** — paired evaluation run (costs money — calls real models). `-f/--from ` · `-t/--to ` · `-c/--config` · `-s/--suite ` · `--suite-name ` · `--resume` · `-y/--yes` @@ -905,6 +920,7 @@ On an interactive terminal, `report` and `compare` end with one line asking for **`evalshift compare`** — full pipeline under one live display: two models, **one** suite. It takes a single `--suite-name`/`--suite`, auto-selects when exactly one suite is wired, and prints a ready-to-run command per suite when several are. It does not run every suite — loop over the names, or use one CI job per suite. Formerly `all`; that name is hidden but still works, since scaffolded `EVALSHIFT.md` files reference it. All `run` flags, plus `--gate` · `--policy-gate` · `--open` · `--push` · `--insights/--no-insights` +With a capture limit set, prints the `suite freshness` findings for a named or auto-selected `source: captured` suite as `⚠` lines above the verdict. ### Hosted @@ -920,6 +936,7 @@ All `run` flags, plus `--gate` · `--policy-gate` · `--open` · `--push` · `-- **`evalshift capture fetch`** — `--suite` · `--since <30m|24h|7d|ISO>` · `-c/--config` (needs `captures.store`) **`evalshift capture promote `** — `--as ` · `--suite` · `--input-var` (default `input`) · `--tag` (repeatable) · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` (default `first`) · `--allow-errored` · `-f/--force` **`evalshift capture sync`** — `--suite` · `--input-var` · `--tag` · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` · `--allow-errored` · `-c/--config` · `-f/--force` · `--write/--print` (default write) · `--keep-duplicates` · `--since` · `--offline` · `--repair` +With `captures.max_age` set, promotes only captures recorded within it on `defaults.source_model`, and its automatic fetch skips older objects (`--since` overrides). With `captures.max_cases` set, also keeps each suite a capped, route-balanced sample, retires stale cases as fresh ones replace them, and prints one `kept / retired / added` line per suite. Reads `evalshift.yaml` under `--offline` too. **`evalshift capture clean [suite]`** — `--promoted` (default) · `--all` · `-y/--yes` **`evalshift capture diff `** diff --git a/docs/configuration.md b/docs/configuration.md index 522b581..e832ae4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -5,7 +5,8 @@ page documents every field — types, defaults, and what they do. `evalshift init` writes a minimal, capture-first `evalshift.yaml` — a passthrough `replay` prompt, default models for the provider picked with -`--provider gemini|openai|anthropic|deepseek` (default `gemini`), evaluators, and an empty +`--provider gemini|openai|anthropic|deepseek` (default `gemini`), evaluators, +`captures:` limits (`max_age: 14d`, `max_cases: 200`), and an empty managed `suites:` block you fill in with `evalshift capture sync`. Below is the canonical reference. @@ -1077,14 +1078,25 @@ pulls new captures down before it promotes them, with no flags to remember: ```yaml captures: store: s3://acme-evals/support-agent + max_age: 14d # optional: see Bounded suites below + max_cases: 200 # optional, per suite ``` -| Field | Type | Default | Description | -| ------- | ------------- | ------- | ----------- | -| `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | +| Field | Type | Default | Description | +| ----------- | --------------------------------- | ------- | ----------- | +| `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | +| `max_age` | duration (`30m`, `24h`, `14d`; > 0) | (none) | Captures recorded longer ago than this never enter a suite, and promoted cases that old are stale: `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | +| `max_cases` | int (≥ 1) | (none) | The most cases `capture sync` keeps in each captured suite. Omit and suites grow with every sync. | -CLI versions before this release reject the `captures:` key (the config is strict), so bump -the CLI — and a pinned GitHub Action's `evalshift-version` — before adding it. +Setting either `max_age` or `max_cases` turns [bounded suites](#bounded-suites) on. Both work +without `store`: they govern how captures become suites, whether the captures came from disk or +from a bucket. A `max_age` that is not a duration fails to load with +`captures.max_age: expected a duration like 30m, 24h or 14d`. A bare number is rejected too, +because it would silently mean seconds. + +CLI versions before 1.3.0 reject the `captures:` key, and versions before this release reject +`max_age` and `max_cases` (the config is strict). Bump the CLI, and a pinned GitHub Action's +`evalshift-version`, before adding them. Accepted URI forms — the same grammar the SDK uses: @@ -1105,8 +1117,166 @@ are immutable and toolsets content-addressed, so "exists" means "current"), skip already promoted into a suite (so `capture clean` never causes a re-download; `capture sync --repair` alone re-fetches the promoted captures its rebuilds need, past `--since` too), and writes each file atomically. `--since 24h` (or `7d`, `30m`, an ISO date) limits a fetch to recently -written captures; `--offline` skips it and works with what is already local. Retention in the bucket -is yours to set with a lifecycle rule; the SDK's `max_captures` does not apply there. +written captures; `--offline` skips it and works with what is already local. With `max_age` set, +`capture sync`'s own fetch skips captures older than it (see [Fetch window](#fetch-window)). + +Retention in the bucket is yours to set with a lifecycle rule; the SDK's `max_captures` does not +apply there, and the CLI never writes to or deletes from the bucket. Expire the `captures/` prefix +after a period ≥ `max_age` (30 days for `max_age: 14d`, say). That bounds storage and how long +production user data sits in the bucket. Leave `toolsets/` alone: committed suites reference +those sidecars. + +```bash +aws s3api put-bucket-lifecycle-configuration --bucket acme-evals \ + --lifecycle-configuration '{"Rules": [{"ID": "expire-evalshift-captures", + "Status": "Enabled", "Filter": {"Prefix": "support-agent/captures/"}, + "Expiration": {"Days": 30}}]}' +``` + +On Google Cloud Storage, use an Object Lifecycle Management rule with a `Delete` action, an +`age` condition and a `matchesPrefix` of `/captures/`. On Azure, use a lifecycle +management policy with `daysAfterModificationGreaterThan` and a `prefixMatch` of +`//captures/`. + +### Bounded suites + +A team that switches `EVALSHIFT_CAPTURE=1` on in production for a few days writes thousands of +captures per agent. Without limits, `capture sync` promotes all of them, and every later sync adds +to the suite. CI then replays thousands of rows on both models, plus the judge. After a production +migration, cases recorded on the old model keep gating CI as if they were still ground truth. +`max_age` and `max_cases` keep each captured suite a capped, route-balanced sample of recent +production on `defaults.source_model`, refreshed by `capture sync` and a reviewed PR. + +**Opt-in.** Every behaviour on this page applies only when `max_age` or `max_cases` is set. +`defaults.source_model` alone changes nothing, since nearly every config sets it. Without the two +keys, sync promotes and keeps exactly what it did before. The one change that applies either way: +promoted case files now record `captured_at` (the capture's `created_at`, verbatim) and +`source_model` (the `model_id` of the capture's first `model_call`, the call a replay re-sends). +Sync backfills both on existing cases whose capture is still local, rewriting nothing else on the +case. The summary line then ends `, backfilled capture time and model on N case(s)`. +`golden.jsonl` does not change. + +**Stale and eligible.** One clock instant is taken per sync. + +- **Too old** (a capture) or **age-stale** (a promoted case): recorded before now − `max_age`. + Only with `max_age` set. A case written before `captured_at` existed takes its age from + `promoted_at`, which is later than the capture: it looks younger than it is and is retired + later, never earlier. A case with neither timestamp is never too old. +- **On another model** (a capture) or **model-stale** (a case): its recorded model is known and is + not `defaults.source_model`. Only with `defaults.source_model` set. Models are compared by bare + name: lower-cased, provider prefix dropped (`anthropic/claude-sonnet-5` → `claude-sonnet-5`), + one trailing snapshot date dropped (`-20260301`, `-2026-08-07`). There is no alias lookup. Set + `defaults.source_model` to the id your app sends, not a friendly alias for it, or the guardrail + below trips. +- **Eligible**: a capture not yet promoted that survives the usual filters (no events, blocked, + errored, duplicate) and is neither too old nor on another model. + +**Guardrail.** For each suite, when at least one of its cases and new captures recorded a model +and none of them was recorded on `defaults.source_model`, sync does not filter that suite by +model and warns: + +```text +⚠ planner: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: claude-sonnet-5-20260301 ×212); not filtering by model — check defaults.source_model +``` + +A typo, an alias, or a migration whose first new-model capture has not arrived yet can neither +empty a suite nor freeze it. + +**The plan** (with `max_cases` set). Sync plans each suite in four steps: + +1. **A conversation is the unit.** The new turns of one conversation enter together or not at + all; a conversation larger than the free slots is skipped and the next one tried. A capture + without a `conversation_id` is its own unit. +2. **Fill free slots, route-balanced.** The free slots are `max_cases` minus the reviewed and fresh + cases already in the suite. A route is the set of tool names a case expects in round 1 (no + tools is a route too). Each pick goes to the least-represented route that still has a + conversation that fits. Within a route, conversations are taken in the order of a hash of their + id, with no randomness: the same captures give the same suite on any machine, and syncing again + with nothing new changes no file. +3. **Retire stale cases to make room, not because they are stale.** A stale case whose question an + added capture asks again is retired outright: the new case replaces it, and the suite never + holds the same content twice. Any other stale case leaves only while the suite is over + `max_cases`: cases recorded on another model first, then the oldest. A suite with nothing to + replace its stale cases keeps them, up to the cap, and `doctor` and `compare` say so (below). +4. **Trim what is still over.** A suite promoted before the limits existed, or after `max_cases` + was lowered, loses fresh cases from its most-represented route, oldest first. A case whose + capture time is unknown counts as the oldest. + +Cases a human marked `provenance: reviewed` on any expected tool call are never retired or +trimmed. If they alone exceed the cap, sync keeps them and warns +(`alpha: 3 reviewed case(s) over captures.max_cases 2; reviewed cases are never retired, so it +stays over.`). + +With `max_cases` set, sync plans every suite directory under `.evalshift/suites/` that holds case +files, even with no new captures, so an over-cap suite is trimmed on the next sync. That includes +suites sync did not build: suites promoted by hand with `capture promote`, and suites whose +`suites:` entry is `managed: false`. `managed: false` freezes the YAML entry, not the case files. +`--suite` narrows the plan to one suite. A suite directory whose case files cannot be read is left +untouched: its `golden.jsonl` and `suites:` entry stay as they are, sync warns +(`alpha: could not read its promoted cases (ValidationError); left it unchanged.`), and the other +suites still sync. + +Duplicates: with `max_cases` set, a stale case that is not reviewed gives up its claim on its +content, so a fresh capture of the same question can replace it, under `--keep-duplicates` too. +With `max_age` alone, nothing is ever retired: too-old and other-model captures are not promoted, +and a fresh capture of a stale case's question is skipped as a duplicate. + +**Output.** The usual summary line is unchanged. Below it, sync prints one line per planned suite: + +```text +✓ promoted 50 capture(s) into 1 suite(s). + planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model +``` + +`kept` counts the cases that stay, `retired` the ones that left (with `N over max_cases` for a +trim), `added N of M eligible` the new cases out of the eligible captures, and `not eligible` the +captures filtered as too old or on another model. Zero counts are left out. With `max_age` alone +the line is only ` planner: not eligible: 812 older than 14d`, printed when something was +filtered. + +#### Fetch window + +With `max_age` set and `store` configured, the automatic fetch that `capture sync` runs skips +objects last written more than `max_age` ago. The summary names the cutoff: +`fetched 40 capture(s) and 0 toolset(s) from s3://… (skipped 4 promoted, 11 older than +captures.max_age (14d))`. `--since` overrides it. `capture list` and `capture fetch` do not apply +it. The fetch does not sample: a listing gives only key, time and size, and a turn without its own +messages list rebuilds its history from sibling captures, so everything inside the window is +downloaded. `EVALSHIFT_SAMPLE_RATE` on the agent governs the volume. On a fresh checkout, the +committed case files tell the fetch which captures are already promoted, so only new ones inside +the window are downloaded. + +If `evalshift.yaml` cannot be read or does not validate, sync runs without the limits. When the +file has a top-level `captures:` key, it says so, online or `--offline`: +`could not read evalshift.yaml (1 schema problem found); syncing without captures.max_age / +captures.max_cases.` + +#### Freshness in `doctor` and `compare` + +CI reads `golden.jsonl` and never touches the bucket, so a stale suite would go unnoticed there. +`doctor` and `compare` read the case files committed beside `golden.jsonl` and say what is stale. +The freshness findings appear only with a limit set, and only for `source: captured` suites. They +measure against `defaults.source_model`, not `--from`. + +- `doctor` adds one `suite freshness` row per captured suite: `ok` + (`alpha: 200 cases, captured 2d–12d ago`), or one `warn` row per finding: + - `alpha: newest case captured 23d ago (captures.max_age 14d); refresh with evalshift capture sync` + - `alpha: 18 of 200 cases recorded on another model than claude-sonnet-5-5; evalshift capture sync replaces them as new captures arrive` + - `alpha: no case recorded on claude-sonnet-5-5 (seen: claude-sonnet-5-20260301 ×200); check defaults.source_model, or record new captures` + - `alpha: 2940 cases, over captures.max_cases 200; evalshift capture sync trims it` + - `alpha: 2 case file(s) could not be read; the counts leave them out` +- `doctor` also adds a `captures limits` `warn` row when `store` is set and `max_cases` is not: + `suites grow with every capture sync; set captures.max_cases and captures.max_age`, or + `…; set captures.max_cases` when `max_age` is already set. Existing configs get this nudge; + `init` scaffolds both limits. +- `compare` prints the same findings as `⚠` lines above the verdict, for the suite it runs when + that suite is named with `--suite-name` or auto-selected as the only wired suite. A suite given + by `--suite ` gets none. + +Neither is ever a failure. The trade-off of route balancing: rate budgets such as +`max_overall_regression_rate` are measured over a balanced sample, not over traffic. A rare tool +path regressing is what a migration gate exists to catch, and a traffic-weighted 200-case sample +would give a 1%-traffic route about two cases. ## Capture lifecycle @@ -1135,6 +1305,9 @@ references. Each deletion asks first; `--yes`/`-y` skips both prompts. under `.evalshift/captures/` into golden suites at `.evalshift/suites//golden.jsonl` **and** injects the resulting `suites:` block into `evalshift.yaml` between the managed marker comments. +With `captures.max_age` or `captures.max_cases` set, it promotes a capped, +route-balanced sample of recent captures instead, and retires stale cases as +fresh ones replace them (see [Bounded suites](#bounded-suites)). Each suite's entry is generated from what that suite's own rows contain, tool evaluators included, so nothing has to be wired by hand: diff --git a/docs/conversations.md b/docs/conversations.md index 8a10ed8..11738dc 100644 --- a/docs/conversations.md +++ b/docs/conversations.md @@ -91,6 +91,10 @@ exchanges are absent), and the system prompt is only recovered when at least one turn in the conversation recorded a full messages list. Prefer the messages-list convention above when you can. +With `captures.max_cases` set, a conversation's new turns enter a suite +together or not at all: one larger than the suite's free slots is skipped, +never split (see [Bounded suites](configuration.md#bounded-suites)). + ```bash evalshift capture sync # promote every capture, # wire suites: into evalshift.yaml diff --git a/docs/sdk.md b/docs/sdk.md index 3ec6487..6cc9c47 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -117,15 +117,28 @@ EVALSHIFT_CAPTURE=1 EVALSHIFT_CAPTURE_STORE=s3://acme-evals/support-agent python The SDK ships each capture (and its toolset sidecar) to the bucket on a background thread, fail-open; see the SDK docs for the `SIGTERM` / Lambda flush note. Bucket retention is a -lifecycle rule; `EVALSHIFT_MAX_CAPTURES` applies to local disk only. +lifecycle rule (expire the `captures/` prefix after a period ≥ `captures.max_age`); +`EVALSHIFT_MAX_CAPTURES` applies to local disk only. `captures.max_age` / `captures.max_cases` +in `evalshift.yaml` keep the suites built from the bucket bounded (see +[configuration](configuration.md#bounded-suites)). ## 3. Promote captures into suites Run the CLI from the directory holding `.evalshift/` (or set `EVALSHIFT_DIR`). -If the captures live in a bucket, name it once in `evalshift.yaml` -(`captures: {store: s3://acme-evals/support-agent}`) and install the provider's client library -(`pip install boto3`, `google-cloud-storage` or `azure-storage-blob azure-identity`); `list` and -`sync` then fetch new captures first. `evalshift capture fetch` does only that step. +If the captures live in a bucket, name it once in `evalshift.yaml` and install the provider's +client library (`pip install boto3`, `google-cloud-storage` or `azure-storage-blob azure-identity`); +`list` and `sync` then fetch new captures first. `evalshift capture fetch` does only that step. + +```yaml +captures: + store: s3://acme-evals/support-agent # optional: where the SDK ships captures + max_age: 14d # only captures from the last 14 days enter a suite + max_cases: 200 # each suite holds at most 200 cases +``` + +`evalshift init` scaffolds the two limits. With them set, `sync` keeps each suite a capped, +route-balanced sample of recent captures on `defaults.source_model`, and replaces stale cases as +fresh captures arrive (see [Bounded suites](configuration.md#bounded-suites)). ```bash evalshift capture list # what was recorded (--json for machine output) diff --git a/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md b/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md index 09784f5..3cf9fc4 100644 --- a/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md +++ b/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md @@ -3080,3 +3080,18 @@ gh pr create --title "feat(capture): bounded capture suites (captures.max_age / ``` The maintainer merges the PR; the release (version bump, tag) is a separate PR. + +## Execution notes + +What shipped differs from the tasks above in these ways (controller rulings during Tasks 1–10; the spec and the user docs describe the shipped behaviour): + +1. Model matching (Task 3): `same_model` compares bare names only (lower-cased, provider prefix dropped, one trailing `-YYYYMMDD` / `-YYYY-MM-DD` dropped). The registry/alias fallback shown in Task 3 was removed, so a `defaults.source_model` spelled as an alias trips the guardrail warning. +2. Planner (Task 6): a stale case whose content an added capture replays is retired outright, even under the cap; other stale cases leave only while `excess > 0`. A case with an unknown capture time counts as oldest when ordering retirement and trim (spec §5 amended). +3. Dedup with limits (Task 8): a stale case gives up its content claim only when `max_cases` is set and the case is not reviewed; with `max_age` alone a fresh re-capture of a stale case's question is skipped as a duplicate. +4. Twin retirement applies under `--keep-duplicates` too. +5. Fetch window (Task 7): only `capture sync`'s automatic fetch follows `max_age`; `capture list` and `capture fetch` do not; `--since` overrides it. +6. Limits warning (Task 7): `could not read {path} ({reason}); syncing without captures.max_age / captures.max_cases.` prints online too, not only under `--offline`. +7. With `max_cases` set, sync also trims suites it did not build: hand-promoted suites and suites whose entry is `managed: false`. Reviewed cases are never retired. +8. A suite directory whose case files cannot be read is left untouched (its `golden.jsonl` and `suites:` entry stay); sync warns and continues. +9. doctor `captures limits` row (Task 9, per spec §7): warns when `captures.store` is set and `captures.max_cases` is not, with `suites grow with every capture sync; set captures.max_cases and captures.max_age`, or `…; set captures.max_cases` when only `max_age` is set. +10. compare freshness lines (Task 9): printed for the suite compare runs when named with `--suite-name` or auto-selected (single wired suite), not for `--suite `; measured against `defaults.source_model`, not `--from`. diff --git a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md index ecf13c0..3b4aa67 100644 --- a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md +++ b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md @@ -158,13 +158,17 @@ route, content keys. `SuitePlan`: groups to add, `(case name, reason)` to retire - **Fill:** `slots = max(0, max_cases − |P| − |F|)`. Repeatedly pick the route with the fewest cases in `P ∪ F ∪ added` among routes that still have a group that fits (ties → route string), add its next group, until slots or groups run out. -- **Retire (replacement only):** `excess = |P| + |F| + |added| + |S| − max_cases`. While - `excess > 0`, retire from `S` in this order: (a) a stale case whose content key equals an added - turn's (its own replacement), (b) model-stale, (c) age-stale oldest first; ties by name. A stale - case is thus only ever retired to make room — **a suite never shrinks because it went stale.** +- **Retire (replacement only):** first, unconditionally, every stale case whose content key equals + an added turn's is retired — its own replacement, even when the suite is under its cap (§6 "no + duplicate content" wins). Then `excess = |P| + |F| + |added| + |S remaining| − max_cases`, and + while `excess > 0`, retire from the remaining `S` in this order: (b) model-stale, (c) age-stale + oldest first; ties by name. A stale case is thus only ever retired to make room or to give way + to its fresh twin — **a suite never shrinks because it went stale.** - **Trim:** if `excess > 0` once `S` is exhausted (a legacy or over-cap suite), retire from `F`: the most-represented route each time, oldest first within it. `P` is never trimmed; if `|P| > max_cases` sync warns and keeps them. +- **Unknown capture time:** when ordering retirement (c) or the trim, a case whose capture time is + unknown (neither `captured_at` nor `promoted_at` parses) counts as the oldest. Without `max_cases`: every eligible candidate is added (today's behaviour, minus too-old / other-model captures when those filters are set), and nothing is retired. diff --git a/llms-full.txt b/llms-full.txt index 48c440a..9c1ff12 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -154,6 +154,9 @@ evalshift init [-f/--force] [-d/--directory DIR] [--ci] [--wire-agents/--no-wire Without --ci, warns after writing when an existing .github/workflows/*.yml pins an older or newer evalshift-version than this CLI, or none (advisory, exit 0; see CI pin drift). --ci writes the pin itself and does not warn about the file it just wrote. + Also scaffolds `captures: {max_age: 14d, max_cases: 200}` (store: commented out) between + migration_policy and the managed region, so new projects get bounded suites (see capture + sync). Existing configs are never rewritten; doctor's `captures limits` row nudges them. PROFILE budgets (regression/critical/equivalence/arg-drift/tool-divergence/ trace-rule-violations/cost/latency; every profile pins trace-rule violations at 0): model-upgrade (default): .30/1/.75/.20/.20/0/.30/.30 (= MigrationPolicy defaults) @@ -195,6 +198,22 @@ evalshift doctor "... (skipped until then)" for an advisory one; one ok "judge and embedding models have API keys" otherwise; no row without a loadable config. compare's doctor stage ignores this row (its own suite-scoped preflight decides). + `captures limits` row: warn (exit 0) when captures.store is set and captures.max_cases is + not -- "suites grow with every capture sync; set captures.max_cases and captures.max_age" + ("...; set captures.max_cases" when max_age is already set): max_age alone never caps a + suite's size. + `suite freshness` rows: ONLY with captures.max_age or captures.max_cases set, one per + `source: captured` entry in suites: whose dir holds case files. Reads the PromotedCase files + committed beside golden.jsonl (captured_at, else promoted_at; source_model). ok "alpha: 200 + cases, captured 2d–12d ago"; else one warn (exit 0, never fail) per finding: + "alpha: newest case captured 23d ago (captures.max_age 14d); refresh with evalshift capture sync" + "alpha: 18 of 200 cases recorded on another model than ; evalshift + capture sync replaces them as new captures arrive" + "alpha: no case recorded on (seen: ×N); check + defaults.source_model, or record new captures" (the guardrail case: nothing matches) + "alpha: 2940 cases, over captures.max_cases 200; evalshift capture sync trims it" + "alpha: N case file(s) could not be read; the counts leave them out" + Measured against defaults.source_model, models matched on bare names (see capture sync). evalshift run [-f/--from MODEL] [-t/--to MODEL] [-c CONFIG] [-s/--suite FILE] [--suite-name NAME] [--resume] [-y/--yes] Paired run over (prompt x example x {source,target}). --from/--to override @@ -237,6 +256,11 @@ evalshift compare [run flags] [--gate SEVS] [--policy-gate] [--open] [--push] doctor -> run -> evaluate -> analyze -> report under one live progress display, for one suite. Auto-selects the suite when exactly one is wired; with several, pass --suite-name (there is no run-every-suite mode -- loop in the shell or use a CI matrix job per suite). + Freshness: with captures.max_age or captures.max_cases set, prints doctor's `suite + freshness` findings as `⚠` lines above the verdict (compare drops doctor warn rows, so it + computes them itself) -- for the suite it runs when named with --suite-name OR auto-selected + (single wired suite) and `source: captured`; NOT for --suite . Measured against + defaults.source_model, not --from. This is how a CI-only team learns its suite is stale. Hosted: evalshift login [--token es_...] [--host URL] [--no-browser] [--timeout SECS=900] @@ -328,6 +352,7 @@ Captures (read from .evalshift/captures/ under CWD, or $EVALSHIFT_DIR; with capt in evalshift.yaml, list/sync first mirror the bucket into it): evalshift capture list [SUITE] [--json] [-c CONFIG] [--offline] With captures.store set, runs fetch first (limited to SUITE when given) unless --offline. + captures.max_age does NOT bound list's fetch (only sync's). With --json, stdout is pure JSON: the fetch summary and every warning go to stderr. evalshift capture fetch [--suite S] [--since 30m|24h|7d|ISO] [-c CONFIG] Mirrors captures.store into : lists toolsets/ then captures/, downloads only objects @@ -348,7 +373,8 @@ evalshift capture fetch [--suite S] [--since 30m|24h|7d|ISO] [-c CONFIG] naming pip install boto3 | google-cloud-storage | azure-storage-blob azure-identity (az:// needs both; one command installs them). No captures.store, or no evalshift.yaml -> exit 1; an invalid evalshift.yaml -> its load error, exit 1; an unreadable or non-UTF-8 one -> a one-line - "could not read " error, exit 1. Has no --offline flag. + "could not read " error, exit 1. Has no --offline flag. captures.max_age does NOT + bound it; only sync's automatic fetch applies the max_age window. evalshift capture promote CAPTURE_ID [--as CASE_ID] [--suite S] [--input-var NAME=input] [--tag T]... [--strict-args] [--names-only] [--tool-count] [--rounds first(default)|all] [--allow-errored] [-f/--force] @@ -396,12 +422,75 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri cannot be rebuilt without its capture: run --repair BEFORE a plain sync on such suites (fresh clone, after capture clean), or a duplicate capture may be promoted -- the duplicate warning after --repair names it. - Promotes EVERY capture: groups by conversation_id, orders by turn_index, one SuiteExample per + Promotes EVERY capture (with captures limits set: a bounded sample, see BOUNDED SUITES below): + groups by conversation_id, orders by turn_index, one SuiteExample per turn; writes .evalshift/suites//golden.jsonl and rewrites the managed suites: block in evalshift.yaml (between ">>> evalshift suites" markers). Content-duplicate captures skipped by default (duplicates inflate n, corrupt paired stats); --keep-duplicates opts out. Dedup is seeded from cases already promoted in the suite dir, so it spans sync runs. Reports "wired generation config for N case(s)" when promoted captures carried one. + BOUNDED SUITES (captures.max_age: duration 30m|24h|14d > 0; captures.max_cases: int >= 1; both + default unset). OPT-IN: every rule below applies only when at least one is set; + defaults.source_model alone changes nothing; without them sync promotes/keeps exactly what it + did before. ALWAYS (limits or not): PromotedCase gains captured_at (envelope created_at, + verbatim) + source_model (first model_call's model_id, the call a replay re-sends); sync + backfills empty ones on existing cases whose capture is local (only those two fields; the + summary adds ", backfilled capture time and model on N case(s)"). SuiteExample/golden.jsonl + unchanged. + Config: sync loads evalshift.yaml under --offline too. Unreadable/invalid -> no limits; when + the file has a top-level captures: key it prints (online AND offline) "could not read + (); syncing without captures.max_age / captures.max_cases." + Staleness (one `now` per sync): too old / age-stale = recorded before now - max_age (case: + captured_at, else promoted_at -- an upper bound, so a legacy case is retired later, never + earlier; neither parseable -> never too old). Other model / model-stale = recorded model + known and != defaults.source_model. Match on BARE names only: lower-case, drop everything up + to the last '/', strip one trailing -YYYYMMDD or -YYYY-MM-DD. NO registry/alias lookup + (forward aliases like gpt-5 -> openai/gpt-4o would conflate distinct models), so a friendly + alias as source_model does not match what the app sends -> guardrail. + Guardrail, per suite: when >= 1 of its cases + unpromoted captures recorded a model and none + matches defaults.source_model, the model filter is OFF for that suite and sync warns + ": none of N case(s) and capture(s) were recorded on (seen: ×N, + ...); not filtering by model — check defaults.source_model". A wrong source_model, or a + rollout before its first new-model capture, never empties or freezes a suite. + Eligible = not yet promoted, passes the usual filters (no events, blocked, errored, dup), not + too old, not other model. max_age alone: ineligible captures are not promoted; nothing is + retired; a fresh re-capture of a stale case's question is skipped as a DUPLICATE. + Plan (max_cases set; pure, deterministic, no RNG): planned suites = those with local captures + PLUS every /suites// holding case files (so an over-cap suite is trimmed with no + new captures; includes suites promoted by hand with `capture promote` and suites whose entry + is managed: false -- that flag freezes the YAML entry, not the case files); --suite narrows. + Kept cases: reviewed (any expected tool call provenance: reviewed -> NEVER retired), fresh, + stale. Unit = conversation group (conversation_id; a lone capture is its own group), added + whole or skipped (next group tried). slots = max_cases - reviewed - fresh; repeatedly add the + next group of the route (sorted round-1 expected tool names; none is its own route) with the + fewest cases so far that still has a group that fits (ties by route name); within a route, + sha256(group key) order. Retire: a stale case whose content key an added turn replays is + retired OUTRIGHT, even under the cap (no duplicate content; also under --keep-duplicates); + other stale cases only while over max_cases: model-stale first, then oldest (unknown time = + oldest), ties by name -- a suite never shrinks because it went stale. Still over: trim fresh + cases from the most-represented route, oldest first. Reviewed over the cap -> kept + ": + N reviewed case(s) over captures.max_cases M; reviewed cases are never retired, so it stays + over." Dedup seed: a stale non-reviewed case gives up its content claim (max_cases set only), + so a fresh re-capture of its question can replace it. Same captures -> same suite on any + machine; re-running with nothing new changes no file. Retired case files are deleted, then + golden.jsonl is rebuilt. A suite whose case files cannot be read is LEFT UNTOUCHED (golden.jsonl + and suites entry as they were): ": could not read its promoted cases (); + left it unchanged." -- other suites still sync. + Output, after the summary line, one line per planned suite (zero counts left out): + " planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), + added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model" + (retired reasons also "N over max_cases"; kept = cases that stay; eligible = candidates after + dedup). max_age alone: " planner: not eligible: 812 older than 14d", only when non-empty. + Fetch window: with max_age set, sync's AUTOMATIC fetch skips objects whose last_modified is + before now - max_age ("(skipped 4 promoted, 11 older than captures.max_age (14d))"); --since + overrides it; capture list / capture fetch do not apply it. No sampling at fetch time (a + listing has key/time/size only; history rebuild needs sibling turns) -- volume inside the + window is EVALSHIFT_SAMPLE_RATE's job. Committed case files supply the promoted-id skip list, + so a fresh checkout downloads only new captures inside the window. + Route balancing is coverage-weighted, not traffic-weighted: rate budgets are measured over a + balanced sample. The CLI never writes to or deletes from the bucket; expire the captures/ + prefix (not toolsets/, which committed suites reference) with a lifecycle rule after a period + >= max_age. After writing (or printing the block to paste) warns on CI pin drift: a workflow step `uses: evalshift/evalshift-action@...` whose evalshift-version is older than this CLI or absent gets a warning naming workflow + job and the `evalshift-version: ""` to set; pins @@ -810,6 +899,17 @@ captures: trailing slash stripped; default none = local disk only. Client libraries are plain packages: pip install boto3 (s3://; AWS_ENDPOINT_URL covers MinIO/R2/B2), google-cloud-storage (gs://), azure-storage-blob azure-identity (az://). Credentials: each provider's default chain only. + max_age: duration|null = null # N + m|h|d (30m, 24h, 14d), > 0; parsed to a timedelta. Bad + # value -> "captures.max_age: expected a duration like 30m, 24h or 14d" (a bare number is + # rejected too: it would silently mean seconds). Captures older than this never enter a suite; + # promoted cases that old are stale. Also sync's fetch window. + max_cases: int>=1|null = null # most cases capture sync keeps in EACH captured suite + # Setting either turns bounded suites on (see capture sync); both valid without store. No + # per-suite override. Not in defaults on purpose: the hosted eval_config_hash hashes defaults + # and does not hash captures. compute_config_hash (resume) covers the whole config, so a + # --resume across the upgrade that added these fields reports a mismatch. Older CLIs reject + # both keys AND case files carrying captured_at/source_model: bump the CLI everywhere capture + # commands run, and the CI action's evalshift-version; CI compare reads golden.jsonl only. Model ids: built-in registry provides aliases/metadata but NEVER gates — LiteLLM is the call-time authority; any LiteLLM-supported model works. Unknown ids pass through with provider @@ -1467,7 +1567,13 @@ content leaves the machine. paired statistics); --keep-duplicates opts out. The dedup key is the built example's replayed content (inputs + history), NOT the envelope input_hash (the SDK salts that with conversation_id and derives it from the agent's bound args). Seeded from already-promoted - cases, so dedup spans sync runs. + cases, so dedup spans sync runs. With captures.max_cases set, a stale non-reviewed case does + not seed it (its fresh re-capture replaces it). +- Bounded capture sync (captures.max_age / captures.max_cases set): a suite never shrinks + because it went stale (stale cases leave only to make room or to be replaced by their fresh + twin); reviewed cases are never retired; a conversation is added whole or not at all; the + plan is deterministic, and syncing again with nothing new changes no file; a + defaults.source_model no capture ran on filters nothing (guardrail warning). - Slices: every distinct example tag is a slice under its own name, plus "all"; each evaluator is analyzed overall AND per slice; per-slice budgets go under migration_policy.slices keyed by the tag and inherit unset fields from the top level. Nothing to configure: there is no top-level @@ -1538,6 +1644,10 @@ migration_policy: {max_overall_regression_rate: 0.30, max_critical_regressions: # ^ the model-upgrade profile = the MigrationPolicy field defaults. init does NOT # scaffold tool_argument_drift_floor; it is inherited at 0.9. Tighter presets: # --profile cost-reduction|quantization|provider-switch|local-model. +captures: # init scaffolds the limits (store: commented out) + max_age: 14d + max_cases: 200 + # store: s3:/// # project: your-org/your-project # hosted only; init scaffolds it commented out suites: {} # init writes the managed region LAST, after # migration_policy: it is the only part a command From 6bc6788e21e4ccca19343c2cb10ced3ee4a18b1f Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:05:39 +0200 Subject: [PATCH 19/26] docs: correct bounded-suite docs against the shipped CLI Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 61 +++++++++++-------- DOCS.md | 10 +-- docs/configuration.md | 30 +++++---- docs/conversations.md | 2 +- docs/getting-started.md | 14 +++-- docs/sdk.md | 2 +- ...026-10-16-bounded-capture-suites-design.md | 12 +++- llms-full.txt | 24 +++++--- 8 files changed, 93 insertions(+), 62 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea68077..30ef8e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,32 +10,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - **Bounded capture suites.** Two optional settings under `captures:`, - `max_age: 14d` and `max_cases: 200`, keep each captured suite a capped, - route-balanced sample of recent production on `defaults.source_model`. - Setting either one turns the feature on. `capture sync` then promotes only - captures recorded within `max_age` on the source model. With `max_cases` - set, it never grows a suite past the cap and swaps stale cases (too old, or - recorded on another model) for fresh captures as they arrive. A stale case - whose question a fresh capture asks again is retired at once. Any other - stale case leaves only to make room, so a suite that has nothing to replace - it with never shrinks. A conversation enters whole or not at all. Cases a - human marked `provenance: reviewed` are never retired. The same captures - give the same suite on any machine, and syncing again with nothing new - changes no file. `max_cases` also trims suites sync did not build: suites - promoted by hand with `capture promote`, and suites whose entry is - `managed: false` (that flag freezes the YAML entry, not the case files). - Model ids match on bare names: lower-cased, provider prefix and trailing - snapshot date dropped, no alias lookup. If none of a suite's cases and - captures was recorded on `defaults.source_model`, sync warns - (`none of N case(s) and capture(s) were recorded on …; not filtering - by model — check defaults.source_model`) and does not filter that suite by - model rather than empty it. `max_age` also bounds what `capture sync`'s - automatic fetch downloads from `captures.store`; `--since` still overrides - it, and `capture list` and `capture fetch` are unchanged. Sync prints one - line per suite: `alpha: 200 cases — kept 150, retired 50 (32 older than - 14d, 18 on another model), added 50 of 2940 eligible; not eligible: …`. - A suite whose case files cannot be read is left untouched, with a warning, - and the other suites still sync. + `max_age: 14d` and `max_cases: 200`, together keep each captured suite a + capped, route-balanced sample of recent production on + `defaults.source_model`. Setting either one turns the feature on; without + them `capture sync` behaves as before. +- **Filter.** With either limit set, `capture sync` promotes only captures + recorded on `defaults.source_model`, and with `max_age` set, only captures + recorded within it. Model ids match on bare names: lower-cased, provider + prefix and trailing snapshot date dropped, no alias lookup. `max_age` alone + caps nothing. `capture promote` ignores the limits. +- **Plan.** With `max_cases` set, sync never grows a suite past the cap and + swaps stale cases (too old, or recorded on another model) for fresh + captures as they arrive. A stale case leaves at once when a fresh capture of + the same question is added in its place; with no free slot nothing is + added and the stale case stays. Any other stale case leaves only to make + room, so a suite that has nothing to replace it with never shrinks. A + conversation enters whole or not at all. Cases a human marked + `provenance: reviewed` are never retired. The same captures give the same + suite on any machine, and syncing again with nothing new changes no file. + `max_cases` also trims suites sync did not build: suites promoted by hand + with `capture promote`, and suites whose entry is `managed: false` (that + flag freezes the YAML entry, not the case files). A suite whose case files + cannot be read is left untouched, with a warning, and the other suites + still sync. +- **Guardrail.** If none of a suite's cases and captures was recorded on + `defaults.source_model`, sync warns (`none of N case(s) and capture(s) were + recorded on …; not filtering by model — check + defaults.source_model`) and does not filter that suite by model rather than + empty it. +- **Fetch.** `max_age` also bounds which captures `capture sync`'s automatic + fetch downloads from `captures.store` (toolset sidecars are always + fetched); `--since` still overrides it, and `capture list` and + `capture fetch` are unchanged. +- **Output.** Sync prints one line per suite: `alpha: 200 cases — kept 150, + retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 + eligible; not eligible: …`. - `doctor` gains `suite freshness` rows for each `source: captured` suite when either limit is set: `ok` (`alpha: 200 cases, captured 2d–12d ago`), or one `warn` per finding — the newest case is older than `max_age`, cases were diff --git a/DOCS.md b/DOCS.md index 65d2439..710e975 100644 --- a/DOCS.md +++ b/DOCS.md @@ -160,7 +160,7 @@ The evalshift-sdk (`pip install evalshift-sdk` — already a dependency of the C ```bash evalshift capture list # table of recorded captures (--json for machine-readable) evalshift capture fetch # (captures.store only) mirror new captures into .evalshift/ -evalshift capture sync # promote ALL captures → suites + wire config +evalshift capture sync # promote captures → suites + wire config evalshift capture promote cap_ab12 --as refund_case_1 # promote one evalshift capture diff cap_ab12 cap_cd34 # compare two tool traces evalshift capture clean # delete promoted capture files + sweep orphaned toolsets @@ -177,7 +177,7 @@ captures: and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download; `capture sync --repair` alone re-fetches the promoted captures its rebuilds need, past `--since` too), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching client library — `pip install boto3`, `google-cloud-storage` or `azure-storage-blob azure-identity` — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, at most its scheme. A missing client library exits 1 naming the `pip install ` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). -**Bounded suites.** `captures.max_age` (a duration: `30m`, `24h`, `14d`) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) whose question a new capture asks again is retired outright; any other stale case leaves only to make room, other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). +**Bounded suites.** Together, `captures.max_age` (a duration: `30m`, `24h`, `14d`) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) is retired outright when a fresh capture of the same question is added in its place (a suite with no free slot adds nothing, so the stale case stays); any other stale case leaves only to make room, other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). **`capture sync`** is the workhorse. Per suite it: @@ -222,7 +222,7 @@ init → doctor → run → evaluate → analy .json (if policy) ``` -- **`doctor`** validates local config and shows which provider keys are visible. Exit 1 only when an existing `evalshift.yaml` fails validation — the row gives the problem count and points at `evalshift validate`, which prints each problem — or when a blocking `semantic` / `llm_judge` model has no API key; other missing keys are soft warnings. Its second row, `evalshift-sdk`, reports the SDK version the `evalshift` import name resolves to in this environment (`warn` when the SDK is missing or shadowed by an older CLI's leftover files; never a failure). It also reports the toolset each configured suite carries (or the flat `golden.jsonl`) and flags a suite whose examples carry more than one distinct toolset — legal (each example dispatches its own), but also the shape a wiring mistake takes. When a workflow under `.github/workflows/` uses the GitHub Action it adds a `ci pin` row: `ok` (`pinned to `) when CI installs this CLI version, `warn` when the pin is older, absent, or newer than the local CLI (see [Pin drift](#pin-drift)). When the config sets `captures.store`, a `captures.store` row reports it: `warn` naming the `pip install ` to run when the client library is missing (the capture commands that need it exit 1 themselves), `warn` when the bucket cannot be listed (no cloud credentials on this machine, say), `ok` (` reachable`) otherwise; no row without a store. When `captures.store` is set and `captures.max_cases` is not, a `captures limits` row warns `suites grow with every capture sync; set captures.max_cases and captures.max_age` (`…; set captures.max_cases` when `max_age` is set). With either limit set, each `source: captured` suite gets a `suite freshness` row: `ok` (`alpha: 200 cases, captured 2d–12d ago`), or one `warn` per finding — newest case older than `max_age`, cases recorded on another model than `defaults.source_model` (or none on it), over `max_cases`, unreadable case files (see [Bounded suites](docs/configuration.md#freshness-in-doctor-and-compare)). When the config wires an `llm_judge` evaluator and names both `defaults.source_model` and `target_model`, a `judge family` row warns for every `judge_model` that resolves to the same provider as an arm (self-preference bias; `ok` "from a third family" otherwise, no row when either arm is unset) — advisory, never a failure; `validate` prints the same line and the report repeats it above the verdict (see [`evaluators.llm_judge`](docs/configuration.md#evaluatorsllm_judge)). When the config has a `semantic` or `llm_judge` evaluator, an `evaluator keys` row checks every judge and embedding model across the top-level block and every named suite's: `fail` (exit 1) for a blocking evaluator with no key, `warn` for an advisory one (skipped until the key is exported), one `ok` row when every such model has a key (see [Missing judge or embedding keys](#missing-judge-or-embedding-keys)). +- **`doctor`** validates local config and shows which provider keys are visible. Exit 1 only when an existing `evalshift.yaml` fails validation — the row gives the problem count and points at `evalshift validate`, which prints each problem — or when a blocking `semantic` / `llm_judge` model has no API key; other missing keys are soft warnings. Its second row, `evalshift-sdk`, reports the SDK version the `evalshift` import name resolves to in this environment (`warn` when the SDK is missing or shadowed by an older CLI's leftover files; never a failure). It also reports the toolset each configured suite carries (or the flat `golden.jsonl`) and flags a suite whose examples carry more than one distinct toolset — legal (each example dispatches its own), but also the shape a wiring mistake takes. When a workflow under `.github/workflows/` uses the GitHub Action it adds a `ci pin` row: `ok` (`pinned to `) when CI installs this CLI version, `warn` when the pin is older, absent, or newer than the local CLI (see [Pin drift](#pin-drift)). When the config sets `captures.store`, a `captures.store` row reports it: `warn` naming the `pip install ` to run when the client library is missing (the capture commands that need it exit 1 themselves), `warn` when the bucket cannot be listed (no cloud credentials on this machine, say), `ok` (` reachable`) otherwise; no row without a store. When `captures.store` is set and `captures.max_cases` is not, a `captures limits` row warns `suites grow with every capture sync; set captures.max_cases and captures.max_age` (`…; set captures.max_cases` when `max_age` is set). With either limit set, each `source: captured` suite gets a `suite freshness` row: `ok` (`alpha: 200 cases, captured 2d–12d ago`), or one `warn` per finding — newest case older than `max_age`, cases recorded on another model than `defaults.source_model` (or none on it), over `max_cases`, unreadable case files (see [Freshness in `doctor` and `compare`](docs/configuration.md#freshness-in-doctor-and-compare)). When the config wires an `llm_judge` evaluator and names both `defaults.source_model` and `target_model`, a `judge family` row warns for every `judge_model` that resolves to the same provider as an arm (self-preference bias; `ok` "from a third family" otherwise, no row when either arm is unset) — advisory, never a failure; `validate` prints the same line and the report repeats it above the verdict (see [`evaluators.llm_judge`](docs/configuration.md#evaluatorsllm_judge)). When the config has a `semantic` or `llm_judge` evaluator, an `evaluator keys` row checks every judge and embedding model across the top-level block and every named suite's: `fail` (exit 1) for a blocking evaluator with no key, `warn` for an advisory one (skipped until the key is exported), one `ok` row when every such model has a key (see [Missing judge or embedding keys](#missing-judge-or-embedding-keys)). - **`run`** parses prompts, validates every example against every prompt, estimates cost, then dispatches `(prompt × example × {source, target})` calls through an async orchestrator under a concurrency semaphore. Responses are cached — per call, and per replayed round for tool-calling examples (see [Response cache](#response-cache)); progress is checkpointed every 50 completions. - **`evaluate`** scores each (source, target) pair with the configured evaluators, one `EvalRecord` per pair × evaluator. Scoring runs under the same `defaults.concurrency` semaphore as `run`, and the embedding/judge calls it makes go through the same response cache. - **`analyze`** runs paired statistics per `(prompt, evaluator, slice)`, applies Benjamini–Hochberg FDR correction, classifies severities, and — when a `migration_policy` is configured — computes a pass/fail verdict. @@ -344,7 +344,7 @@ suites: {} | `migration_policy` | block \| absent | Regression budgets, see [Migration policy](#migration-policy-and-ci-gating) | | `suites` | map | Named suites (`{name: {source: captured\|jsonl, path: ..., evaluators: ..., managed: true}}`); the block between the `>>> evalshift suites` markers is managed by `capture sync`. See [Per-suite evaluators](#per-suite-evaluators) | | `retention` | block | `max_runs_per_suite` (default 20, `0` disables), `run_ttl_days` (default off) | -| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. `max_age` (duration, `30m`/`24h`/`14d`) and `max_cases` (int ≥ 1), both default unset: setting either keeps each captured suite a capped, route-balanced sample of recent production on `defaults.source_model` (bounded suites). See [Capturing from production](#capturing-from-production) | +| `captures` | block | `store`: `s3://`, `gs://` or `az://` URI of the object store the SDK ships captures to; `sync` and `list` mirror it locally first. Omit for local disk only. `max_age` (duration, `30m`/`24h`/`14d`) and `max_cases` (int ≥ 1), both default unset. Setting either turns bounded suites on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite; with either set, captures recorded on another model than `defaults.source_model` are filtered too. See [Capturing from production](#capturing-from-production) | `thresholds` was a top-level field until it was removed. It was free-form and gated nothing — `migration_policy` is the single source of truth for gating — so rather than being silently ignored it is now rejected by name, and a config that still carries it fails to load: @@ -936,7 +936,7 @@ With a capture limit set, prints the `suite freshness` findings for a named or a **`evalshift capture fetch`** — `--suite` · `--since <30m|24h|7d|ISO>` · `-c/--config` (needs `captures.store`) **`evalshift capture promote `** — `--as ` · `--suite` · `--input-var` (default `input`) · `--tag` (repeatable) · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` (default `first`) · `--allow-errored` · `-f/--force` **`evalshift capture sync`** — `--suite` · `--input-var` · `--tag` · `--strict-args` · `--names-only` · `--tool-count` · `--rounds {first,all}` · `--allow-errored` · `-c/--config` · `-f/--force` · `--write/--print` (default write) · `--keep-duplicates` · `--since` · `--offline` · `--repair` -With `captures.max_age` set, promotes only captures recorded within it on `defaults.source_model`, and its automatic fetch skips older objects (`--since` overrides). With `captures.max_cases` set, also keeps each suite a capped, route-balanced sample, retires stale cases as fresh ones replace them, and prints one `kept / retired / added` line per suite. Reads `evalshift.yaml` under `--offline` too. +With `captures.max_age` or `captures.max_cases` set, promotes only captures recorded on `defaults.source_model`. With `captures.max_age` set, also promotes only captures recorded within it, and its automatic fetch skips older captures (`--since` overrides). With `captures.max_cases` set, also keeps each suite a capped, route-balanced sample, retires stale cases as fresh ones replace them, and prints one `kept / retired / added` line per suite. Reads `evalshift.yaml` under `--offline` too. **`evalshift capture clean [suite]`** — `--promoted` (default) · `--all` · `-y/--yes` **`evalshift capture diff `** diff --git a/docs/configuration.md b/docs/configuration.md index e832ae4..1a25a9b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1085,14 +1085,15 @@ captures: | Field | Type | Default | Description | | ----------- | --------------------------------- | ------- | ----------- | | `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | -| `max_age` | duration (`30m`, `24h`, `14d`; > 0) | (none) | Captures recorded longer ago than this never enter a suite, and promoted cases that old are stale: `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | +| `max_age` | duration (`30m`, `24h`, `14d`; > 0) | (none) | Captures recorded longer ago than this never enter a suite through `capture sync` (`capture promote` ignores the limits), and promoted cases that old are stale: `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | | `max_cases` | int (≥ 1) | (none) | The most cases `capture sync` keeps in each captured suite. Omit and suites grow with every sync. | Setting either `max_age` or `max_cases` turns [bounded suites](#bounded-suites) on. Both work without `store`: they govern how captures become suites, whether the captures came from disk or -from a bucket. A `max_age` that is not a duration fails to load with -`captures.max_age: expected a duration like 30m, 24h or 14d`. A bare number is rejected too, -because it would silently mean seconds. +from a bucket. A `max_age` that is not a duration fails to load; `evalshift validate` reports it as +`captures.max_age: Value error, expected a duration like 30m, 24h or 14d`, followed by the value +it got (`, got '2w'`). A bare number such as `14` is rejected the same way, because it would +silently mean seconds. CLI versions before 1.3.0 reject the `captures:` key, and versions before this release reject `max_age` and `max_cases` (the config is strict). Bump the CLI, and a pinned GitHub Action's @@ -1147,7 +1148,7 @@ migration, cases recorded on the old model keep gating CI as if they were still `max_age` and `max_cases` keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`, refreshed by `capture sync` and a reviewed PR. -**Opt-in.** Every behaviour on this page applies only when `max_age` or `max_cases` is set. +**Opt-in.** Every behaviour in this section applies only when `max_age` or `max_cases` is set. `defaults.source_model` alone changes nothing, since nearly every config sets it. Without the two keys, sync promotes and keeps exactly what it did before. The one change that applies either way: promoted case files now record `captured_at` (the capture's `created_at`, verbatim) and @@ -1196,11 +1197,12 @@ empty a suite nor freeze it. 3. **Retire stale cases to make room, not because they are stale.** A stale case whose question an added capture asks again is retired outright: the new case replaces it, and the suite never holds the same content twice. Any other stale case leaves only while the suite is over - `max_cases`: cases recorded on another model first, then the oldest. A suite with nothing to + `max_cases`: cases recorded on another model first, then the oldest (among the other-model + cases, one whose capture time is unknown counts as the oldest). A suite with nothing to replace its stale cases keeps them, up to the cap, and `doctor` and `compare` say so (below). 4. **Trim what is still over.** A suite promoted before the limits existed, or after `max_cases` - was lowered, loses fresh cases from its most-represented route, oldest first. A case whose - capture time is unknown counts as the oldest. + was lowered, loses fresh cases from its most-represented route, oldest first. A fresh case + whose capture time is unknown counts as the oldest. Cases a human marked `provenance: reviewed` on any expected tool call are never retired or trimmed. If they alone exceed the cap, sync keeps them and warns @@ -1237,7 +1239,8 @@ filtered. #### Fetch window With `max_age` set and `store` configured, the automatic fetch that `capture sync` runs skips -objects last written more than `max_age` ago. The summary names the cutoff: +captures last written more than `max_age` ago. Toolset sidecars missing locally are always +fetched, whatever their age. The summary names the cutoff: `fetched 40 capture(s) and 0 toolset(s) from s3://… (skipped 4 promoted, 11 older than captures.max_age (14d))`. `--since` overrides it. `capture list` and `capture fetch` do not apply it. The fetch does not sample: a listing gives only key, time and size, and a turn without its own @@ -1287,7 +1290,7 @@ the base). The CLI consumes them via the `capture` commands: ```shell evalshift capture list # see what the SDK recorded evalshift capture fetch # (captures.store only) mirror new captures locally -evalshift capture sync --input-var query # promote every capture + wire suites: +evalshift capture sync --input-var query # promote captures + wire suites: evalshift run --suite-name support_agent --yes # score a candidate model against it evalshift capture clean # prune already-promoted captures ``` @@ -1305,9 +1308,10 @@ references. Each deletion asks first; `--yes`/`-y` skips both prompts. under `.evalshift/captures/` into golden suites at `.evalshift/suites//golden.jsonl` **and** injects the resulting `suites:` block into `evalshift.yaml` between the managed marker comments. -With `captures.max_age` or `captures.max_cases` set, it promotes a capped, -route-balanced sample of recent captures instead, and retires stale cases as -fresh ones replace them (see [Bounded suites](#bounded-suites)). +With `captures.max_age` set, captures older than it are not promoted; with +`captures.max_cases` set, it promotes a capped, route-balanced sample instead, +and retires stale cases as fresh ones replace them (see +[Bounded suites](#bounded-suites)). Each suite's entry is generated from what that suite's own rows contain, tool evaluators included, so nothing has to be wired by hand: diff --git a/docs/conversations.md b/docs/conversations.md index 11738dc..f8f6058 100644 --- a/docs/conversations.md +++ b/docs/conversations.md @@ -96,7 +96,7 @@ together or not at all: one larger than the suite's free slots is skipped, never split (see [Bounded suites](configuration.md#bounded-suites)). ```bash -evalshift capture sync # promote every capture, +evalshift capture sync # promote captures, # wire suites: into evalshift.yaml evalshift run --suite-name support_agent --yes # score a candidate model against it ``` diff --git a/docs/getting-started.md b/docs/getting-started.md index 80b1ed7..4aefc46 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -75,8 +75,9 @@ passthrough `replay` prompt, an advisory LLM-judge evaluator and an advisory semantic evaluator (Anthropic and DeepSeek have no embeddings endpoint, so their scaffolds borrow OpenAI's or Gemini's, whichever key you already have; without it, `compare` skips `semantic` and names the key to -export), an empty managed `suites:` block for `capture sync` to fill, and the migration -policy. `init` refuses to clobber an existing `evalshift.yaml`; pass +export), the migration policy, `captures:` limits (`max_age: 14d` and +`max_cases: 200`, with `store:` commented out), and an empty managed +`suites:` block for `capture sync` to fill. `init` refuses to clobber an existing `evalshift.yaml`; pass `--force` to overwrite, or `--directory my-eval/` to scaffold into a different folder. @@ -145,9 +146,12 @@ instrument the agent, write `golden.jsonl` by hand instead — see evalshift capture sync ``` -`capture sync` promotes every recorded capture into -`.evalshift/suites//golden.jsonl` and injects the matching -`suites:` block into `evalshift.yaml`. See +`capture sync` promotes a capped sample of recent captures — at most +`captures.max_cases` per suite, recorded within `captures.max_age` on +`defaults.source_model` — into `.evalshift/suites//golden.jsonl` +and injects the matching `suites:` block into `evalshift.yaml`. Later syncs +replace stale cases as fresh captures arrive; see +[Bounded suites](configuration.md#bounded-suites). See [Configuration](configuration.md) for the full capture lifecycle. If a workflow under `.github/workflows/` pins an older or newer CLI than the one you just synced with, or none at all, it ends with an advisory warning and the diff --git a/docs/sdk.md b/docs/sdk.md index 6cc9c47..749770c 100644 --- a/docs/sdk.md +++ b/docs/sdk.md @@ -143,7 +143,7 @@ fresh captures arrive (see [Bounded suites](configuration.md#bounded-suites)). ```bash evalshift capture list # what was recorded (--json for machine output) evalshift capture fetch # mirror new captures from captures.store -evalshift capture sync # promote ALL captures → suites + wire evalshift.yaml +evalshift capture sync # promote captures → suites + wire evalshift.yaml evalshift capture promote cap_ab12 --as refund_case_1 evalshift capture diff cap_ab12 cap_cd34 evalshift capture clean # delete already-promoted capture files diff --git a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md index 3b4aa67..95e0abd 100644 --- a/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md +++ b/docs/superpowers/specs/2026-10-16-bounded-capture-suites-design.md @@ -2,6 +2,13 @@ **Date:** 2026-10-16 · **Status:** approach A approved in chat, spec for maintainer review · **Repos:** evalshift-cli (behaviour + docs); evalshift-sdk, evalshift-action, evalshift-server, evalshift-client (nothing — follow-ups recorded) +> **Implemented; this spec predates the code.** Where the shipped behaviour differs, the +> deviations are listed in the plan's "Execution notes" +> (`docs/superpowers/plans/2026-10-16-bounded-capture-suites.md`). Example output strings below +> (§3, §6, §7, §8) were written before implementation and are not the CLI's exact text. The +> source and the user docs (`docs/configuration.md#bounded-suites`, `DOCS.md`, `llms-full.txt`) +> are authoritative. + ## Problem A team that ships captures to a bucket (`EVALSHIFT_CAPTURE_STORE`) and switches @@ -167,8 +174,9 @@ route, content keys. `SuitePlan`: groups to add, `(case name, reason)` to retire - **Trim:** if `excess > 0` once `S` is exhausted (a legacy or over-cap suite), retire from `F`: the most-represented route each time, oldest first within it. `P` is never trimmed; if `|P| > max_cases` sync warns and keeps them. -- **Unknown capture time:** when ordering retirement (c) or the trim, a case whose capture time is - unknown (neither `captured_at` nor `promoted_at` parses) counts as the oldest. +- **Unknown capture time:** a case whose capture time is unknown (neither `captured_at` nor + `promoted_at` parses) counts as the oldest when ordering the model-stale group (b) and the trim. + An age-stale case (c) always has a known time. Without `max_cases`: every eligible candidate is added (today's behaviour, minus too-old / other-model captures when those filters are set), and nothing is retired. diff --git a/llms-full.txt b/llms-full.txt index 9c1ff12..6f27111 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -466,9 +466,10 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri fewest cases so far that still has a group that fits (ties by route name); within a route, sha256(group key) order. Retire: a stale case whose content key an added turn replays is retired OUTRIGHT, even under the cap (no duplicate content; also under --keep-duplicates); - other stale cases only while over max_cases: model-stale first, then oldest (unknown time = - oldest), ties by name -- a suite never shrinks because it went stale. Still over: trim fresh - cases from the most-represented route, oldest first. Reviewed over the cap -> kept + ": + no free slot -> nothing added -> the stale twin stays. Other stale cases only while over + max_cases: model-stale first (oldest first, unknown time = oldest), then age-stale oldest + first, ties by name -- a suite never shrinks because it went stale. Still over: trim fresh + cases from the most-represented route, oldest first (unknown time = oldest). Reviewed over the cap -> kept + ": N reviewed case(s) over captures.max_cases M; reviewed cases are never retired, so it stays over." Dedup seed: a stale non-reviewed case gives up its content claim (max_cases set only), so a fresh re-capture of its question can replace it. Same captures -> same suite on any @@ -481,8 +482,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model" (retired reasons also "N over max_cases"; kept = cases that stay; eligible = candidates after dedup). max_age alone: " planner: not eligible: 812 older than 14d", only when non-empty. - Fetch window: with max_age set, sync's AUTOMATIC fetch skips objects whose last_modified is - before now - max_age ("(skipped 4 promoted, 11 older than captures.max_age (14d))"); --since + Fetch window: with max_age set, sync's AUTOMATIC fetch skips CAPTURES whose last_modified is + before now - max_age; toolset sidecars missing locally are always fetched ("(skipped 4 promoted, 11 older than captures.max_age (14d))"); --since overrides it; capture list / capture fetch do not apply it. No sampling at fetch time (a listing has key/time/size only; history rebuild needs sibling turns) -- volume inside the window is EVALSHIFT_SAMPLE_RATE's job. Committed case files supply the promoted-id skip list, @@ -900,11 +901,16 @@ captures: pip install boto3 (s3://; AWS_ENDPOINT_URL covers MinIO/R2/B2), google-cloud-storage (gs://), azure-storage-blob azure-identity (az://). Credentials: each provider's default chain only. max_age: duration|null = null # N + m|h|d (30m, 24h, 14d), > 0; parsed to a timedelta. Bad - # value -> "captures.max_age: expected a duration like 30m, 24h or 14d" (a bare number is - # rejected too: it would silently mean seconds). Captures older than this never enter a suite; - # promoted cases that old are stale. Also sync's fetch window. + # value -> validate prints "captures.max_age: Value error, expected a duration like 30m, + # 24h or 14d" (+ ", got '2w'" for a string; a bare number is rejected too: it would + # silently mean seconds). Captures older than this never enter a suite through capture + # sync (capture promote ignores the limits); promoted cases that old are stale. Also sync's + # fetch window. max_cases: int>=1|null = null # most cases capture sync keeps in EACH captured suite - # Setting either turns bounded suites on (see capture sync); both valid without store. No + # Setting either turns bounded suites on (see capture sync): max_age filters what enters a + # suite and marks old cases stale; max_cases caps and route-balances each suite; with either + # set, captures/cases on another model than defaults.source_model are filtered/stale too. + # max_age alone caps nothing. Both valid without store. No # per-suite override. Not in defaults on purpose: the hosted eval_config_hash hashes defaults # and does not hash captures. compute_config_hash (resume) covers the whole config, so a # --resume across the upgrade that added these fields reports a mismatch. Older CLIs reject From 90717866ec6003d8e4efc2305d059409fcaa7f04 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:12:55 +0200 Subject: [PATCH 20/26] docs: a stale twin leaves only when its fresh copy is added or to make room Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++-- DOCS.md | 2 +- docs/configuration.md | 13 +++--- .../2026-10-16-bounded-capture-suites.md | 2 +- llms-full.txt | 41 ++++++++++--------- src/evalshift_cli/config/models.py | 5 ++- 6 files changed, 38 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30ef8e8..d7784ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `max_age: 14d` and `max_cases: 200`, together keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`. Setting either one turns the feature on; without - them `capture sync` behaves as before. + them `capture sync` keeps and promotes exactly what it did before. - **Filter.** With either limit set, `capture sync` promotes only captures recorded on `defaults.source_model`, and with `max_age` set, only captures recorded within it. Model ids match on bare names: lower-cased, provider @@ -22,9 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Plan.** With `max_cases` set, sync never grows a suite past the cap and swaps stale cases (too old, or recorded on another model) for fresh captures as they arrive. A stale case leaves at once when a fresh capture of - the same question is added in its place; with no free slot nothing is - added and the stale case stays. Any other stale case leaves only to make - room, so a suite that has nothing to replace it with never shrinks. A + the same question is added in its place; if its fresh copy is not added, + the stale case leaves only to make room, like any other stale case. So a + suite that has nothing to replace its stale cases with never shrinks. A conversation enters whole or not at all. Cases a human marked `provenance: reviewed` are never retired. The same captures give the same suite on any machine, and syncing again with nothing new changes no file. diff --git a/DOCS.md b/DOCS.md index 710e975..736b19d 100644 --- a/DOCS.md +++ b/DOCS.md @@ -177,7 +177,7 @@ captures: and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download; `capture sync --repair` alone re-fetches the promoted captures its rebuilds need, past `--since` too), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching client library — `pip install boto3`, `google-cloud-storage` or `azure-storage-blob azure-identity` — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, at most its scheme. A missing client library exits 1 naming the `pip install ` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). -**Bounded suites.** Together, `captures.max_age` (a duration: `30m`, `24h`, `14d`) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) is retired outright when a fresh capture of the same question is added in its place (a suite with no free slot adds nothing, so the stale case stays); any other stale case leaves only to make room, other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). +**Bounded suites.** Together, `captures.max_age` (a duration: `30m`, `24h`, `14d`) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) is retired outright when a fresh capture of the same question is added in its place; if its fresh copy is not added, the stale case leaves only to make room, like any other stale case. Stale cases that leave to make room go other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). **`capture sync`** is the workhorse. Per suite it: diff --git a/docs/configuration.md b/docs/configuration.md index 1a25a9b..aba22ee 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1085,15 +1085,15 @@ captures: | Field | Type | Default | Description | | ----------- | --------------------------------- | ------- | ----------- | | `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | -| `max_age` | duration (`30m`, `24h`, `14d`; > 0) | (none) | Captures recorded longer ago than this never enter a suite through `capture sync` (`capture promote` ignores the limits), and promoted cases that old are stale: `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | +| `max_age` | duration (`30m`, `24h`, `14d`; > 0) | (none) | Captures recorded longer ago than this never enter a suite through `capture sync` (`capture promote` ignores the limits), and promoted cases that old are stale: with `max_cases` also set, `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | | `max_cases` | int (≥ 1) | (none) | The most cases `capture sync` keeps in each captured suite. Omit and suites grow with every sync. | Setting either `max_age` or `max_cases` turns [bounded suites](#bounded-suites) on. Both work without `store`: they govern how captures become suites, whether the captures came from disk or from a bucket. A `max_age` that is not a duration fails to load; `evalshift validate` reports it as -`captures.max_age: Value error, expected a duration like 30m, 24h or 14d`, followed by the value -it got (`, got '2w'`). A bare number such as `14` is rejected the same way, because it would -silently mean seconds. +`captures.max_age: Value error, expected a duration like 30m, 24h or 14d`, followed for a string by +the value it got (`, got '2w'`). A bare number such as `14` is rejected too, without that suffix, +because it would silently mean seconds. CLI versions before 1.3.0 reject the `captures:` key, and versions before this release reject `max_age` and `max_cases` (the config is strict). Bump the CLI, and a pinned GitHub Action's @@ -1308,8 +1308,9 @@ references. Each deletion asks first; `--yes`/`-y` skips both prompts. under `.evalshift/captures/` into golden suites at `.evalshift/suites//golden.jsonl` **and** injects the resulting `suites:` block into `evalshift.yaml` between the managed marker comments. -With `captures.max_age` set, captures older than it are not promoted; with -`captures.max_cases` set, it promotes a capped, route-balanced sample instead, +With `captures.max_age` or `captures.max_cases` set, captures recorded on another +model than `defaults.source_model` are not promoted; with `captures.max_age` set, +neither are captures older than it; with `captures.max_cases` set, it promotes a capped, route-balanced sample instead, and retires stale cases as fresh ones replace them (see [Bounded suites](#bounded-suites)). diff --git a/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md b/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md index 3cf9fc4..92d230e 100644 --- a/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md +++ b/docs/superpowers/plans/2026-10-16-bounded-capture-suites.md @@ -3086,7 +3086,7 @@ The maintainer merges the PR; the release (version bump, tag) is a separate PR. What shipped differs from the tasks above in these ways (controller rulings during Tasks 1–10; the spec and the user docs describe the shipped behaviour): 1. Model matching (Task 3): `same_model` compares bare names only (lower-cased, provider prefix dropped, one trailing `-YYYYMMDD` / `-YYYY-MM-DD` dropped). The registry/alias fallback shown in Task 3 was removed, so a `defaults.source_model` spelled as an alias trips the guardrail warning. -2. Planner (Task 6): a stale case whose content an added capture replays is retired outright, even under the cap; other stale cases leave only while `excess > 0`. A case with an unknown capture time counts as oldest when ordering retirement and trim (spec §5 amended). +2. Planner (Task 6): a stale case whose content an added capture replays is retired outright, even under the cap; other stale cases leave only while `excess > 0`. A case with an unknown capture time counts as oldest when ordering the model-stale retirement group and the trim; an age-stale case always has a known time (spec §5 amended). 3. Dedup with limits (Task 8): a stale case gives up its content claim only when `max_cases` is set and the case is not reviewed; with `max_age` alone a fresh re-capture of a stale case's question is skipped as a duplicate. 4. Twin retirement applies under `--keep-duplicates` too. 5. Fetch window (Task 7): only `capture sync`'s automatic fetch follows `max_age`; `capture list` and `capture fetch` do not; `--since` overrides it. diff --git a/llms-full.txt b/llms-full.txt index 6f27111..20947e4 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -422,7 +422,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri cannot be rebuilt without its capture: run --repair BEFORE a plain sync on such suites (fresh clone, after capture clean), or a duplicate capture may be promoted -- the duplicate warning after --repair names it. - Promotes EVERY capture (with captures limits set: a bounded sample, see BOUNDED SUITES below): + Promotes EVERY capture (with captures limits set: filtered by age/model, and capped with + max_cases -- see BOUNDED SUITES below): groups by conversation_id, orders by turn_index, one SuiteExample per turn; writes .evalshift/suites//golden.jsonl and rewrites the managed suites: block in evalshift.yaml (between ">>> evalshift suites" markers). Content-duplicate captures skipped by @@ -465,29 +466,31 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri next group of the route (sorted round-1 expected tool names; none is its own route) with the fewest cases so far that still has a group that fits (ties by route name); within a route, sha256(group key) order. Retire: a stale case whose content key an added turn replays is - retired OUTRIGHT, even under the cap (no duplicate content; also under --keep-duplicates); - no free slot -> nothing added -> the stale twin stays. Other stale cases only while over - max_cases: model-stale first (oldest first, unknown time = oldest), then age-stale oldest - first, ties by name -- a suite never shrinks because it went stale. Still over: trim fresh - cases from the most-represented route, oldest first (unknown time = oldest). Reviewed over the cap -> kept + ": - N reviewed case(s) over captures.max_cases M; reviewed cases are never retired, so it stays - over." Dedup seed: a stale non-reviewed case gives up its content claim (max_cases set only), - so a fresh re-capture of its question can replace it. Same captures -> same suite on any - machine; re-running with nothing new changes no file. Retired case files are deleted, then - golden.jsonl is rebuilt. A suite whose case files cannot be read is LEFT UNTOUCHED (golden.jsonl - and suites entry as they were): ": could not read its promoted cases (); - left it unchanged." -- other suites still sync. + retired OUTRIGHT, even under the cap (no duplicate content; also under --keep-duplicates); if + its fresh copy is not added, the stale case leaves only to make room, like any other stale + case. Other stale cases only while over max_cases: model-stale first (oldest first, unknown + time = oldest), then age-stale oldest first, ties by name -- a suite never shrinks because it + went stale. Still over: trim fresh cases from the most-represented route, oldest first + (unknown time = oldest). Reviewed over the cap -> kept + ": N reviewed case(s) over + captures.max_cases M; reviewed cases are never retired, so it stays over." Dedup seed: a stale + non-reviewed case gives up its content claim (max_cases set only), so a fresh re-capture of + its question can replace it. Same captures -> same suite on any machine; re-running with + nothing new changes no file. Retired case files are deleted, then golden.jsonl is rebuilt. A + suite whose case files cannot be read is LEFT UNTOUCHED (golden.jsonl and suites entry as they + were): ": could not read its promoted cases (); left it unchanged." -- other + suites still sync. Output, after the summary line, one line per planned suite (zero counts left out): " planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model" (retired reasons also "N over max_cases"; kept = cases that stay; eligible = candidates after dedup). max_age alone: " planner: not eligible: 812 older than 14d", only when non-empty. Fetch window: with max_age set, sync's AUTOMATIC fetch skips CAPTURES whose last_modified is - before now - max_age; toolset sidecars missing locally are always fetched ("(skipped 4 promoted, 11 older than captures.max_age (14d))"); --since - overrides it; capture list / capture fetch do not apply it. No sampling at fetch time (a - listing has key/time/size only; history rebuild needs sibling turns) -- volume inside the - window is EVALSHIFT_SAMPLE_RATE's job. Committed case files supply the promoted-id skip list, - so a fresh checkout downloads only new captures inside the window. + before now - max_age; toolset sidecars missing locally are always fetched ("(skipped 4 + promoted, 11 older than captures.max_age (14d))"); --since overrides it; capture list / + capture fetch do not apply it. No sampling at fetch time (a listing has key/time/size only; + history rebuild needs sibling turns) -- volume inside the window is EVALSHIFT_SAMPLE_RATE's + job. Committed case files supply the promoted-id skip list, so a fresh checkout downloads only + new captures inside the window. Route balancing is coverage-weighted, not traffic-weighted: rate budgets are measured over a balanced sample. The CLI never writes to or deletes from the bucket; expire the captures/ prefix (not toolsets/, which committed suites reference) with a lifecycle rule after a period @@ -1699,7 +1702,7 @@ evalshift init --provider gemini # minimal config only pip install evalshift-sdk # already a dependency of the CLI; alone in a capture-only agent EVALSHIFT_CAPTURE=1 python my_agent.py # record captures via SDK instrumentation evalshift capture list -evalshift capture sync # promote all -> .evalshift/suites//golden.jsonl +evalshift capture sync # promote -> .evalshift/suites//golden.jsonl evalshift compare --suite-name --to gemini-3.1-pro-preview --gate critical,high --policy-gate # Tools from code — no separate sync step: capture what your agent actually calls diff --git a/src/evalshift_cli/config/models.py b/src/evalshift_cli/config/models.py index 063dcca..5b1573b 100644 --- a/src/evalshift_cli/config/models.py +++ b/src/evalshift_cli/config/models.py @@ -849,8 +849,9 @@ class CapturesConfig(_StrictModel): default chain, never from this file; a URI carrying ``@`` or ``?`` is rejected. ``None`` (the default) means local disk only. max_age: Captures recorded longer ago than this (``30m``, ``24h``, ``14d``) never enter a - suite, and promoted cases that old are stale: ``capture sync`` replaces them as fresh - captures arrive. Also the fetch window. ``None`` (the default): no age limit. + suite through ``capture sync``, and promoted cases that old are stale: with + ``max_cases`` also set, ``capture sync`` replaces them as fresh captures arrive. Also + the fetch window. ``None`` (the default): no age limit. max_cases: The most cases ``capture sync`` keeps in each captured suite. ``None`` (the default): suites grow with every sync. Setting either limit turns bounded suites on. """ From 46141d361d59ba558e12ef6a374a779737c6179d Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:34:19 +0200 Subject: [PATCH 21/26] fix(config): reject captures.max_age above 36500d A huge max_age ("999999d", a plausible way to write "no limit") passed validation, then overflowed the cutoff capture sync, doctor and compare compute from it. The validator now caps it at 100 years and says to omit max_age for no age limit. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/config/models.py | 12 +++++++++++- tests/unit/test_config_captures.py | 16 ++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/evalshift_cli/config/models.py b/src/evalshift_cli/config/models.py index 5b1573b..121251f 100644 --- a/src/evalshift_cli/config/models.py +++ b/src/evalshift_cli/config/models.py @@ -31,7 +31,7 @@ from evalshift_cli.captures.store_uri import STORE_URI_FORMS, parse_store_uri from evalshift_cli.suite.tags import RESERVED_SLICE_NAME -from evalshift_cli.utils.duration import DURATION_FORMS, parse_duration +from evalshift_cli.utils.duration import DURATION_FORMS, format_duration, parse_duration # Default judge model used across the config when none is specified. # Centralised here so a single edit propagates everywhere. Gemini's @@ -825,6 +825,11 @@ class Retention(_StrictModel): run_ttl_days: int | None = Field(default=None, ge=1) +#: The largest ``captures.max_age``: a cutoff further back than this would overflow ``datetime``. +#: "No age limit" is written by leaving ``max_age`` out. +MAX_AGE_LIMIT = timedelta(days=36500) + + def _duration_from_yaml(value: object) -> object: """``captures.max_age`` is written ``14d``; a bare number would silently mean seconds.""" if value is None or isinstance(value, timedelta): @@ -865,6 +870,11 @@ class CapturesConfig(_StrictModel): def _positive_max_age(cls, value: timedelta | None) -> timedelta | None: if value is not None and value <= timedelta(0): raise ValueError(f"must be greater than 0 (expected a duration like {DURATION_FORMS})") + if value is not None and value > MAX_AGE_LIMIT: + raise ValueError( + f"must be at most {format_duration(MAX_AGE_LIMIT)} (100 years); " + "omit captures.max_age for no age limit" + ) return value @field_validator("store") diff --git a/tests/unit/test_config_captures.py b/tests/unit/test_config_captures.py index 13f18a2..a61750b 100644 --- a/tests/unit/test_config_captures.py +++ b/tests/unit/test_config_captures.py @@ -118,6 +118,22 @@ def test_max_age_rejects_anything_but_a_positive_duration(value: object) -> None EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": value}) +@pytest.mark.parametrize("value", ["999999d", "36501d", "876025h"]) +def test_max_age_rejects_more_than_a_hundred_years(value: str) -> None: + # "No age limit" is written by leaving max_age out; a huge value would overflow the + # cutoff sync, doctor and compare compute from it. + with pytest.raises(ValidationError, match="max_age") as info: + EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": value}) + message = str(info.value) + assert "at most 36500d" in message + assert "omit captures.max_age for no age limit" in message + + +def test_max_age_accepts_a_hundred_years() -> None: + cfg = EvalShiftConfig(prompts=[_PROMPT], captures={"max_age": "36500d"}) + assert cfg.captures.max_age == timedelta(days=36500) + + @pytest.mark.parametrize("value", [0, -5]) def test_max_cases_must_be_positive(value: int) -> None: with pytest.raises(ValidationError, match="max_cases"): From cfa1a58f209ef2c4fb1dbf585b3b6d7f37ecb8ef Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:34:20 +0200 Subject: [PATCH 22/26] test(captures): a review mark in a later round pins the case Co-Authored-By: Claude Opus 5.5 --- tests/unit/test_captures_staleness.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/unit/test_captures_staleness.py b/tests/unit/test_captures_staleness.py index 3272b04..fbdf25d 100644 --- a/tests/unit/test_captures_staleness.py +++ b/tests/unit/test_captures_staleness.py @@ -81,6 +81,21 @@ def test_is_reviewed_reads_any_expectation() -> None: assert not is_reviewed(_example()) +def test_is_reviewed_reads_every_round() -> None: + # A mark in a later round pins the case too: is_reviewed guards deletion. + captured = ExpectedToolCall(tool_name="a", provenance="captured") + reviewed = ExpectedToolCall(tool_name="b", provenance="reviewed") + example = SuiteExample( + id="c", + toolset_ref=_REF, + expected_tools=[captured], + expected_tool_rounds=[[captured], [reviewed]], + ) + assert is_reviewed(example) + unreviewed = example.model_copy(update={"expected_tool_rounds": [[captured], [captured]]}) + assert not is_reviewed(unreviewed) + + def test_route_is_the_sorted_union_of_expected_tool_names() -> None: assert examples_route([_example("b", "a")]) == "a,b" assert examples_route([_example("a"), _example("c")]) == "a,c" From fa4be9c9284e161eeb9a846cc0b674e704c9e529 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:40:36 +0200 Subject: [PATCH 23/26] fix(capture): retire the file a case was read from, never one named after it capture sync planned and deleted bounded-suite cases by their "name" field. A copied case file that kept its name redirected a retirement to the original (deleting a reviewed case), and a renamed file was never deleted while every sync still reported it retired. - iter_promoted_case_files returns each case with its path; sync plans by file stem and retire_promoted_case(path) deletes exactly that file. A deletion that fails is reported and not counted in the suite line. - Backfilled provenance is rewritten in the file it was read from. - A capture whose case file name is already taken is never planned; a planned write that fails is not counted as added, and sync re-plans around what was written so its stale twin stays. - Dedup seeds the content of every case that stays, so a replaceable case that shares content with a reviewed one cannot let a fresh copy in. - A suite synced only because its directory holds case files is rebuilt but wired only when it already has a suites: entry. Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/promote.py | 57 ++++-- src/evalshift_cli/cli/commands/capture.py | 185 +++++++++++------- tests/unit/test_captures_promote.py | 33 +++- tests/unit/test_cli_capture_limits.py | 216 ++++++++++++++++++++++ 4 files changed, 407 insertions(+), 84 deletions(-) diff --git a/src/evalshift_cli/captures/promote.py b/src/evalshift_cli/captures/promote.py index e1125fb..d7009ed 100644 --- a/src/evalshift_cli/captures/promote.py +++ b/src/evalshift_cli/captures/promote.py @@ -1520,6 +1520,11 @@ def promoted_suite_dir(suite: str, *, base: Path | None = None) -> Path: return suites_root(base) / _safe_segment(suite) +def promoted_case_path(case: PromotedCase, *, base: Path | None = None) -> Path: + """Where :func:`write_promoted_case` writes ``case``: ``/suites//.json``.""" + return promoted_suite_dir(case.suite, base=base) / f"{_safe_segment(case.name)}.json" + + def write_promoted_case( case: PromotedCase, *, @@ -1531,15 +1536,23 @@ def write_promoted_case( Raises: FileExistsError: if the target exists and ``force`` is False. """ - suite_dir = promoted_suite_dir(case.suite, base=base) - suite_dir.mkdir(parents=True, exist_ok=True) - path = suite_dir / f"{_safe_segment(case.name)}.json" + path = promoted_case_path(case, base=base) + path.parent.mkdir(parents=True, exist_ok=True) if path.exists() and not force: raise FileExistsError(path) - path.write_text(case.model_dump_json(indent=2) + "\n", encoding="utf-8") + rewrite_promoted_case(path, case) return path +def rewrite_promoted_case(path: Path, case: PromotedCase) -> None: + """Write ``case`` over the case file at ``path``, wherever its ``name`` would put it. + + For a case read from ``path`` (:func:`iter_promoted_case_files`): a file a user renamed, + or copied without changing ``name``, is updated where it lives, never next to it. + """ + path.write_text(case.model_dump_json(indent=2) + "\n", encoding="utf-8") + + def capture_provenance(envelope: CaptureEnvelope) -> tuple[str, str]: """``(captured_at, source_model)`` for a case promoted from ``envelope``. @@ -1550,15 +1563,18 @@ def capture_provenance(envelope: CaptureEnvelope) -> tuple[str, str]: return envelope.created_at, first.model_id if first is not None else "" -def retire_promoted_case(suite: str, name: str, *, base: Path | None = None) -> bool: - """Delete ``/suites//.json``; ``False`` when it was already gone. +def retire_promoted_case(path: Path) -> bool: + """Delete the case file at ``path``; ``False`` when nothing was deleted. - The caller rebuilds ``golden.jsonl`` afterwards (:func:`rebuild_golden_jsonl`). + ``path`` is the file the case was read from (:func:`iter_promoted_case_files`), never one + derived from its ``name``: a copied or renamed case file names another file, and deleting + by name would remove that one instead. ``False`` covers a file already gone and one that + could not be deleted. The caller rebuilds ``golden.jsonl`` afterwards + (:func:`rebuild_golden_jsonl`). """ - path = promoted_suite_dir(suite, base=base) / f"{_safe_segment(name)}.json" try: path.unlink() - except FileNotFoundError: + except OSError: return False return True @@ -1582,20 +1598,32 @@ def example_content_key(example: SuiteExample) -> str: ) -def iter_promoted_cases(suite_dir: Path) -> list[PromotedCase]: - """Load every promoted case file in ``suite_dir`` (``golden.jsonl`` excluded). +def iter_promoted_case_files(suite_dir: Path) -> list[tuple[Path, PromotedCase]]: + """Load every promoted case file in ``suite_dir`` with the path it was read from. + + The path, not the case's ``name``, is what identifies a case on disk: a renamed file keeps + its old ``name``, and two copied files can share one. Raises: pydantic.ValidationError: if a case file is not a valid ``PromotedCase``. """ - cases: list[PromotedCase] = [] + cases: list[tuple[Path, PromotedCase]] = [] for path in sorted(suite_dir.glob("*.json")): if path.name == GOLDEN_FILENAME: continue - cases.append(PromotedCase.model_validate_json(path.read_text(encoding="utf-8"))) + cases.append((path, PromotedCase.model_validate_json(path.read_text(encoding="utf-8")))) return cases +def iter_promoted_cases(suite_dir: Path) -> list[PromotedCase]: + """Load every promoted case file in ``suite_dir`` (``golden.jsonl`` excluded). + + Raises: + pydantic.ValidationError: if a case file is not a valid ``PromotedCase``. + """ + return [case for _, case in iter_promoted_case_files(suite_dir)] + + def rebuild_golden_jsonl(suite_dir: Path) -> Path: """Regenerate ``/golden.jsonl`` from every promoted case file. @@ -1624,10 +1652,13 @@ def rebuild_golden_jsonl(suite_dir: Path) -> Path: "capture_provenance", "duplicate_turn_warnings", "example_content_key", + "iter_promoted_case_files", "iter_promoted_cases", + "promoted_case_path", "promoted_suite_dir", "rebuild_golden_jsonl", "recover_from_messages", "retire_promoted_case", + "rewrite_promoted_case", "write_promoted_case", ] diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index df9a38c..fe6b666 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -38,11 +38,14 @@ capture_provenance, duplicate_turn_warnings, example_content_key, + iter_promoted_case_files, iter_promoted_cases, + promoted_case_path, promoted_suite_dir, rebuild_golden_jsonl, recover_from_messages, retire_promoted_case, + rewrite_promoted_case, write_promoted_case, ) from evalshift_cli.captures.reader import ( @@ -74,7 +77,7 @@ stringified_content, stringified_warnings, ) -from evalshift_cli.captures.selection import CandidateGroup, KeptCase, SuitePlan, plan_suite +from evalshift_cli.captures.selection import CandidateGroup, KeptCase, RetireReason, plan_suite from evalshift_cli.captures.staleness import ( StalenessRules, StaleReason, @@ -249,8 +252,10 @@ def _promoted_content_owners( return owners -def _load_promoted(suite: str, *, base: Path | None) -> tuple[list[PromotedCase], str | None]: - """The suite's promoted cases, plus the error's type name when its files could not be read. +def _load_promoted( + suite: str, *, base: Path | None +) -> tuple[list[tuple[Path, PromotedCase]], str | None]: + """The suite's promoted cases with their files, plus the error's type name on a read failure. A suite with no directory yet has no cases. Only the exception's type name is kept: its text can carry raw bytes of a case file. @@ -259,7 +264,7 @@ def _load_promoted(suite: str, *, base: Path | None) -> tuple[list[PromotedCase] if not suite_dir.is_dir(): return [], None try: - return iter_promoted_cases(suite_dir), None + return iter_promoted_case_files(suite_dir), None except (ValueError, OSError) as exc: # pydantic's ValidationError is a ValueError return [], type(exc).__name__ @@ -282,31 +287,25 @@ def _with_provenance(case: PromotedCase, envelope: CaptureEnvelope) -> PromotedC def _backfill_provenance( - cases: list[PromotedCase], + cases: list[tuple[Path, PromotedCase]], envelopes: Mapping[str, CaptureEnvelope], - *, - base: Path | None, -) -> tuple[list[PromotedCase], int]: +) -> tuple[list[tuple[Path, PromotedCase]], int]: """Fill ``captured_at`` / ``source_model`` on cases written before the fields existed. Only an empty field is filled, only from the case's own capture, and only while that - capture is local; nothing else on the case changes (``promoted_at`` included). Returns - the cases as they now stand and how many files were rewritten. + capture is local; nothing else on the case changes (``promoted_at`` included), and each + case is rewritten in the file it was read from. Returns the cases as they now stand and + how many files were rewritten. """ - out: list[PromotedCase] = [] + out: list[tuple[Path, PromotedCase]] = [] rewritten = 0 - for case in cases: + for path, case in cases: envelope = envelopes.get(case.from_capture) - if envelope is None: - out.append(case) - continue - filled = _with_provenance(case, envelope) - if filled == case: - out.append(case) - continue - write_promoted_case(filled, base=base, force=True) - out.append(filled) - rewritten += 1 + filled = case if envelope is None else _with_provenance(case, envelope) + if filled != case: + rewrite_promoted_case(path, filled) + rewritten += 1 + out.append((path, filled)) return out, rewritten @@ -386,25 +385,27 @@ def _reason_counts(counts: Counter[str], *, max_age: timedelta | None) -> str: def _plan_line( suite: str, - plan: SuitePlan | None, *, kept: int, + retired: Sequence[RetireReason] | None, + added: int, eligible: int, ineligible: Counter[str], max_age: timedelta | None, ) -> str | None: - """The per-suite summary line (spec §6); ``plan`` is ``None`` with ``max_age`` alone.""" + """The per-suite summary line (spec §6), from the files sync actually deleted and wrote. + + ``retired`` is ``None`` with ``max_age`` alone (nothing is planned then). + """ not_eligible = _reason_counts(ineligible, max_age=max_age) - if plan is None: + if retired is None: return f" {suite}: not eligible: {not_eligible}" if not_eligible else None - retired: Counter[str] = Counter(reason for _, reason in plan.retire) - detail = _reason_counts(retired, max_age=max_age) - stayed = kept - len(plan.retire) + detail = _reason_counts(Counter(retired), max_age=max_age) + stayed = kept - len(retired) return ( - f" {suite}: {stayed + plan.added_cases} cases — kept {stayed}, " - f"retired {len(plan.retire)}" + f" {suite}: {stayed + added} cases — kept {stayed}, retired {len(retired)}" + (f" ({detail})" if detail else "") - + f", added {plan.added_cases} of {eligible} eligible" + + f", added {added} of {eligible} eligible" + (f"; not eligible: {not_eligible}" if not_eligible else "") ) @@ -1410,6 +1411,8 @@ def capture_sync( plan_lines: list[str] = [] backed_by = selection.cases_by_capture() suites_seen: set[str] = set() + # Suites planned only because their directory holds case files: no capture this run. + directory_only: set[str] = set() # (suite, content key) -> the name of the case that owns that content, # promoted either by an earlier run (seeded from disk below) or by this one # (where the name is always the capture id). @@ -1446,8 +1449,8 @@ def capture_sync( ): console.print(f"[yellow]⚠[/yellow] {suite_name}: {warning}") envelopes = {r.envelope.capture_id: r.envelope for r in suite_records} - kept_cases, unreadable = _load_promoted(suite_name, base=base) - recorded_names = {c.suite for c in kept_cases} + kept, unreadable = _load_promoted(suite_name, base=base) + recorded_names = {c.suite for _, c in kept} if not suite_records and len(recorded_names) == 1: # A suite planned from its directory alone goes by the name its cases record # (``team/support``, not its directory ``team_support``), as its suites entry does. @@ -1456,8 +1459,10 @@ def capture_sync( suites_seen.discard(suite_name) suites_seen.add(recorded_name) suite_name = recorded_name + if not suite_records: + directory_only.add(suite_name) if unreadable is None: - kept_cases, rewritten = _backfill_provenance(kept_cases, envelopes, base=base) + kept, rewritten = _backfill_provenance(kept, envelopes) backfilled += rewritten if unreadable is not None and limits.active: console.print( @@ -1468,11 +1473,13 @@ def capture_sync( # other suites still sync. suites_seen.discard(suite_name) continue - kept_from = {c.from_capture for c in kept_cases} - stale_by_name: dict[str, StaleReason | None] = {} + # A kept case is its file: a copied or renamed case file can carry another's name. + kept_paths = {path for path, _ in kept} + kept_from = {c.from_capture for _, c in kept} + stale_by_path: dict[Path, StaleReason | None] = {} rules: StalenessRules | None = None if limits.active: - recorded = [c.source_model for c in kept_cases] + [ + recorded = [c.source_model for _, c in kept] + [ capture_provenance(r.envelope)[1] for r in suite_records if r.envelope.capture_id not in kept_from @@ -1480,21 +1487,21 @@ def capture_sync( rules = _suite_rules( suite_name, limits, now=now, recorded_models=recorded, console=console ) - stale_by_name = { - c.name: rules.stale_reason(case_instant(c), c.source_model) for c in kept_cases + stale_by_path = { + path: rules.stale_reason(case_instant(c), c.source_model) for path, c in kept } # Only the planner retires a stale twin, and never a reviewed one; any other stale # case keeps its content, or a fresh copy would be promoted beside it. replaceable = ( { - c.name - for c in kept_cases - if stale_by_name.get(c.name) is not None and not is_reviewed(c.example) + path + for path, c in kept + if stale_by_path.get(path) is not None and not is_reviewed(c.example) } if limits.max_cases is not None else set() ) - replaceable_from = {c.from_capture for c in kept_cases if c.name in replaceable} + replaceable_from = {c.from_capture for path, c in kept if path in replaceable} ineligible: Counter[str] = Counter() pending: list[PromotedCase] = [] built_pairs = build_conversation_examples(suite_records, opts, base=base) @@ -1509,9 +1516,17 @@ def capture_sync( owners = _promoted_content_owners(suite_name, base=base, console=console) # A replaceable stale case does not claim its content: the same question captured # fresh replaces it (spec §5, rule 3). + replaceable_names = {c.name for path, c in kept if path in replaceable} seen_examples.update( - (key, owner) for key, owner in owners.items() if owner not in replaceable + (key, owner) for key, owner in owners.items() if owner not in replaceable_names ) + # ...but a case that stays claims its content even when a replaceable case sharing + # it was the one recorded as its owner, or the fresh copy would duplicate it. + for path, case in kept: + if path not in replaceable: + seen_examples.setdefault( + (suite_name, example_content_key(case.example)), case.name + ) # A repair candidate's rebuilt content belongs to the candidate, whichever capture # is built first: the disk seed above only holds its stringified content, so # without this claim a capture replaying the same turn would promote as a new case. @@ -1656,6 +1671,12 @@ def capture_sync( example=built.example, ) if limits.max_cases is not None and envelope.capture_id not in kept_from: + # A case file already at its path (another capture's case saved under this + # name) would be overwritten or fail to write: never planned as an addition. + target = promoted_case_path(case, base=base) + if target in kept_paths or (target.exists() and not force): + skipped_existing += 1 + continue pending.append(case) continue try: @@ -1670,22 +1691,23 @@ def capture_sync( _warn_repaired_duplicates(suite_name, repaired_here, base=base, console=console) suite_line: str | None = None if limits.max_cases is not None: - plan = plan_suite( - [ - KeptCase( - name=c.name, - instant=case_instant(c), - stale=stale_by_name.get(c.name), - reviewed=is_reviewed(c.example), - route=examples_route([c.example]), - content_key=example_content_key(c.example), - ) - for c in kept_cases - ], - _candidate_groups(pending), - max_cases=limits.max_cases, - ) + # Planned by file stem, which is unique in the directory, so a retirement deletes + # the very file the case was read from. + by_stem = {path.stem: path for path, _ in kept} + kept_plan = [ + KeptCase( + name=path.stem, + instant=case_instant(c), + stale=stale_by_path.get(path), + reviewed=is_reviewed(c.example), + route=examples_route([c.example]), + content_key=example_content_key(c.example), + ) + for path, c in kept + ] + plan = plan_suite(kept_plan, _candidate_groups(pending), max_cases=limits.max_cases) chosen = {cid for group in plan.add for cid in group.capture_ids} + written: list[PromotedCase] = [] for case in pending: if case.name not in chosen: continue @@ -1694,11 +1716,24 @@ def capture_sync( except FileExistsError: skipped_existing += 1 continue + written.append(case) promoted += 1 if case.example.generation_config: wired_generation += 1 - for name, _reason in plan.retire: - retire_promoted_case(suite_name, name, base=base) + if len(written) < plan.added_cases: + # Re-plan around what was written: a stale twin, or a case retired to make + # room, stays when the capture meant to replace it was not added. + plan = plan_suite(kept_plan, _candidate_groups(written), max_cases=limits.max_cases) + retired: list[RetireReason] = [] + for stem, why in plan.retire: + if retire_promoted_case(by_stem[stem]): + retired.append(why) + else: + console.print( + f"[yellow]⚠[/yellow] {escape(suite_name)}: could not delete " + f"{escape(by_stem[stem].name)}; not counted as retired.", + soft_wrap=True, + ) if plan.reviewed_over_cap: console.print( f"[yellow]⚠[/yellow] {escape(suite_name)}: {plan.reviewed_over_cap} " @@ -1707,8 +1742,9 @@ def capture_sync( ) suite_line = _plan_line( suite_name, - plan, - kept=len(kept_cases), + kept=len(kept), + retired=retired, + added=len(written), eligible=len(pending), ineligible=ineligible, max_age=limits.max_age, @@ -1716,8 +1752,9 @@ def capture_sync( elif limits.active: suite_line = _plan_line( suite_name, - None, - kept=len(kept_cases), + kept=len(kept), + retired=None, + added=0, eligible=0, ineligible=ineligible, max_age=limits.max_age, @@ -1794,11 +1831,21 @@ def capture_sync( config_exists = config_path.exists() config_text = config_path.read_text(encoding="utf-8") if config_exists else "" + existing = parse_suites_region(config_text) + # A suite synced only because its directory holds case files is rebuilt, but wired only + # when it already has an entry: a team that took it out of suites: meant it. + wire_paths = { + name: path + for name, path in suite_paths.items() + if name not in directory_only or name in existing + } + if not wire_paths: + return entries, frozen = _build_suite_entries( - suite_paths, + wire_paths, suite_evaluators, config_path=config_path, - existing=parse_suites_region(config_text), + existing=existing, ) suites_yaml = render_suites_yaml(entries) for name in sorted(frozen): @@ -1835,8 +1882,8 @@ def capture_sync( return config_path.write_text(updated, encoding="utf-8") - console.print(f"[green]✓[/green] wired {len(suite_paths)} suite(s) into {config_path}") - first = sorted(suite_paths)[0] + console.print(f"[green]✓[/green] wired {len(wire_paths)} suite(s) into {config_path}") + first = sorted(wire_paths)[0] console.print(f"run: [cyan]evalshift compare --suite-name {first} --to [/cyan]") _warn_ci_pin(console, config_path) diff --git a/tests/unit/test_captures_promote.py b/tests/unit/test_captures_promote.py index 083dfda..4104016 100644 --- a/tests/unit/test_captures_promote.py +++ b/tests/unit/test_captures_promote.py @@ -28,6 +28,7 @@ capture_provenance, duplicate_turn_warnings, example_content_key, + iter_promoted_case_files, rebuild_golden_jsonl, retire_promoted_case, write_promoted_case, @@ -3031,8 +3032,36 @@ def test_deletes_the_case_file(self, tmp_path: Path) -> None: example=SuiteExample(id="cap_1", tools=[]), ) path = write_promoted_case(case, base=tmp_path) - assert retire_promoted_case("alpha", "cap_1", base=tmp_path) is True + assert retire_promoted_case(path) is True assert not path.exists() def test_an_absent_case_is_not_an_error(self, tmp_path: Path) -> None: - assert retire_promoted_case("alpha", "cap_nope", base=tmp_path) is False + assert retire_promoted_case(tmp_path / "suites" / "alpha" / "cap_nope.json") is False + + def test_deletes_only_the_file_it_is_given_when_two_share_a_name(self, tmp_path: Path) -> None: + case = PromotedCase( + name="cap_1", + suite="alpha", + from_capture="cap_1", + example=SuiteExample(id="cap_1", tools=[]), + ) + original = write_promoted_case(case, base=tmp_path) + copy = original.with_name("copy.json") + copy.write_bytes(original.read_bytes()) + [(_, first), (_, second)] = iter_promoted_case_files(original.parent) + assert first.name == second.name == "cap_1" + + assert retire_promoted_case(copy) is True + assert original.exists() + assert not copy.exists() + + def test_iter_promoted_case_files_pairs_each_case_with_its_file(self, tmp_path: Path) -> None: + case = PromotedCase( + name="cap_1", + suite="alpha", + from_capture="cap_1", + example=SuiteExample(id="cap_1", tools=[]), + ) + path = write_promoted_case(case, base=tmp_path) + renamed = path.rename(path.with_name("refund_case.json")) + assert iter_promoted_case_files(path.parent) == [(renamed, case)] diff --git a/tests/unit/test_cli_capture_limits.py b/tests/unit/test_cli_capture_limits.py index 8fa5075..51a23f8 100644 --- a/tests/unit/test_cli_capture_limits.py +++ b/tests/unit/test_cli_capture_limits.py @@ -12,6 +12,7 @@ from typer.testing import CliRunner from evalshift_cli.cli.commands import capture as capture_module +from evalshift_cli.cli.commands._suites import parse_suites_region from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config from evalshift_cli.cli.main import app from evalshift_cli.suite.loader import load_jsonl @@ -499,3 +500,218 @@ def test_force_replaces_a_stale_twin_whatever_the_capture_order( _sync(tmp_path, "--force") assert _cases(tmp_path) == {"b_new"} + + +def _mark_reviewed(path: Path) -> None: + data = json.loads(path.read_text(encoding="utf-8")) + data["example"]["expected_tools"] = [{"tool_name": "lookup", "provenance": "reviewed"}] + path.write_text(json.dumps(data), encoding="utf-8") + + +def test_a_copied_case_file_never_redirects_a_retirement_to_the_reviewed_original( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_cases=4) + _at(monkeypatch, 12) + _sync(tmp_path) + suite = tmp_path / "suites" / "alpha" + original = suite / "cap_0.json" + # A copy whose question was edited but whose "name" still says cap_0. + variant = json.loads(original.read_text(encoding="utf-8")) + variant["example"]["id"] = "variant" + variant["example"]["inputs"] = {"input": "a variant question?"} + (suite / "variant.json").write_text(json.dumps(variant), encoding="utf-8") + _mark_reviewed(original) + _config(tmp_path / "evalshift.yaml", max_cases=3) + + result = _sync(tmp_path) + + assert original.exists() + assert "reviewed" in original.read_text(encoding="utf-8") + assert len(_cases(tmp_path)) == 3 + assert "alpha: 3 cases — kept 3, retired 1 (1 over max_cases), added 0 of 0 eligible" in ( + _flat(result) + ) + + +def test_a_renamed_case_file_is_retired_from_where_it_lives( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + suite = tmp_path / "suites" / "alpha" + (suite / "cap_0.json").rename(suite / "refund_case.json") + _config(tmp_path / "evalshift.yaml", max_cases=2) + + first = _sync(tmp_path) + second = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 2 + assert "alpha: 2 cases — kept 2, retired 1 (1 over max_cases)" in _flat(first) + assert "alpha: 2 cases — kept 2, retired 0," in _flat(second) + + +def test_a_retirement_that_deletes_nothing_is_not_counted( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _config(tmp_path / "evalshift.yaml", max_cases=2) + monkeypatch.setattr(capture_module, "retire_promoted_case", lambda *a, **k: False) + + result = _sync(tmp_path) + + assert len(_cases(tmp_path)) == 3 + flat = _flat(result) + assert "alpha: 3 cases — kept 3, retired 0, added 0 of 0 eligible" in flat + assert "alpha: could not delete cap_0.json; not counted as retired." in flat + + +def test_a_capture_whose_case_name_is_taken_neither_adds_nor_retires_its_twin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _put(tmp_path, "old", text="where is order 1?") + _put(tmp_path, "other", text="something else?") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + suite = tmp_path / "suites" / "alpha" + # `other`'s case now lives under the name a later capture is given. + moved = json.loads((suite / "other.json").read_text(encoding="utf-8")) + moved["name"] = "new" + (suite / "other.json").unlink() + (suite / "new.json").write_text(json.dumps(moved), encoding="utf-8") + _put(tmp_path, "new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"old", "new"} + assert json.loads((suite / "new.json").read_text(encoding="utf-8"))["from_capture"] == "other" + flat = _flat(result) + assert "alpha: 2 cases — kept 2, retired 0, added 0 of 0 eligible" in flat + assert "skipped 3 already-promoted" in flat # old, other, and new (its name is taken) + + +def test_a_write_that_fails_neither_counts_as_added_nor_retires_its_twin( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _put(tmp_path, "old", text="where is order 1?") + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 12) + _sync(tmp_path) + _put(tmp_path, "new", text="where is order 1?", created_at=WEEK_3) + _at(monkeypatch, 30) + real_write = capture_module.write_promoted_case + + def write(case: Any, **kwargs: Any) -> Path: + if case.name == "new": # another process wrote it first + raise FileExistsError(case.name) + return real_write(case, **kwargs) + + monkeypatch.setattr(capture_module, "write_promoted_case", write) + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"old"} + flat = _flat(result) + assert "alpha: 1 cases — kept 1, retired 0, added 0 of 1 eligible" in flat + assert "promoted 0 capture(s)" in flat + + +def test_a_renamed_legacy_case_is_backfilled_in_place( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(2): + _put(tmp_path, f"cap_{i}") + _config(tmp_path / "evalshift.yaml", max_cases=5) + _at(monkeypatch, 12) + _sync(tmp_path) + suite = tmp_path / "suites" / "alpha" + legacy = json.loads((suite / "cap_0.json").read_text(encoding="utf-8")) + legacy.pop("captured_at") + legacy.pop("source_model") + (suite / "cap_0.json").unlink() + (suite / "refund_case.json").write_text(json.dumps(legacy), encoding="utf-8") + + result = _sync(tmp_path) + + assert _cases(tmp_path) == {"refund_case", "cap_1"} + assert json.loads((suite / "refund_case.json").read_text(encoding="utf-8"))["captured_at"] + assert "backfilled capture time and model on 1 case(s)" in _flat(result) + + +def test_a_duplicate_kept_beside_a_replaceable_case_still_blocks_a_fresh_copy( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Two cases share their content (kept with --keep-duplicates): a_old is replaceable, + # b_old is reviewed. A fresh copy would duplicate b_old, which never leaves. + _put(tmp_path, "a_old", text="where is order 1?") + _put(tmp_path, "b_old", text="where is order 1?") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path, "--keep-duplicates") + _mark_reviewed(tmp_path / "suites" / "alpha" / "b_old.json") + _put(tmp_path, "c_new", text="where is order 1?", created_at=WEEK_3) + _config(tmp_path / "evalshift.yaml") + _at(monkeypatch, 30) + + result = _sync(tmp_path) + + assert "c_new" not in _cases(tmp_path) + assert "skipped 1 duplicate capture(s)" in _flat(result) + + +def _wired(tmp_path: Path) -> set[str]: + text = (tmp_path / "evalshift.yaml").read_text(encoding="utf-8") + return set(parse_suites_region(text)) + + +def test_a_suite_dir_without_captures_is_not_wired_back_in( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"a_{i}", suite="alpha") + _put(tmp_path, f"o_{i}", suite="old") + _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + # The team drops `old`: its captures and its suites entry go, its directory stays. + for path in (tmp_path / "captures" / "old").glob("*.json"): + path.unlink() + _config(tmp_path / "evalshift.yaml", max_cases=2) + + _sync(tmp_path) + + assert _wired(tmp_path) == {"alpha"} + assert len(_cases(tmp_path, "old")) == 2 + assert len(load_jsonl(tmp_path / "suites" / "old" / "golden.jsonl").examples) == 2 + + +def test_a_wired_suite_without_captures_stays_wired( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for i in range(3): + _put(tmp_path, f"a_{i}", suite="alpha") + _put(tmp_path, f"o_{i}", suite="old") + config = _config(tmp_path / "evalshift.yaml", max_age=None, max_cases=None) + _at(monkeypatch, 12) + _sync(tmp_path) + for path in (tmp_path / "captures" / "old").glob("*.json"): + path.unlink() + config.write_text( + config.read_text(encoding="utf-8") + "\ncaptures:\n max_cases: 2\n", encoding="utf-8" + ) + + _sync(tmp_path) + + assert _wired(tmp_path) == {"alpha", "old"} + assert len(_cases(tmp_path, "old")) == 2 From ea5b24cea27a0b0a51b6660903b03934e1c54185 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:42:28 +0200 Subject: [PATCH 24/26] fix(capture): freshness advice and sync help match what sync does - A case recorded on another model is replaced only with max_cases set; without it the finding says to set captures.max_cases. - An over-cap suite is trimmed only down to its reviewed cases; the finding says so when they alone exceed max_cases. - capture sync --help describes bounded suites instead of "promote every capture". - The doctor row comment names the real trigger (store without max_cases). Co-Authored-By: Claude Opus 5.5 --- src/evalshift_cli/captures/freshness.py | 21 ++++++++--- src/evalshift_cli/cli/commands/capture.py | 9 ++++- src/evalshift_cli/cli/commands/doctor.py | 2 +- tests/unit/test_captures_freshness.py | 44 +++++++++++++++++++++-- tests/unit/test_cli_capture_limits.py | 8 +++++ 5 files changed, 74 insertions(+), 10 deletions(-) diff --git a/src/evalshift_cli/captures/freshness.py b/src/evalshift_cli/captures/freshness.py index 685347b..77bbe68 100644 --- a/src/evalshift_cli/captures/freshness.py +++ b/src/evalshift_cli/captures/freshness.py @@ -12,7 +12,7 @@ from typing import Literal from evalshift_cli.captures.models import PromotedCase -from evalshift_cli.captures.staleness import case_instant +from evalshift_cli.captures.staleness import case_instant, is_reviewed from evalshift_cli.models.matching import describe_models, model_filter_applies, same_model from evalshift_cli.utils.duration import format_duration @@ -110,12 +110,17 @@ def suite_freshness( if model_filter_applies(models, source_model): other = sum(1 for m in models if m and not same_model(m, source_model)) if other: + # Sync retires a stale case only to make room under max_cases. + advice = ( + "evalshift capture sync replaces them as new captures arrive" + if max_cases is not None + else "set captures.max_cases for evalshift capture sync to replace them" + ) findings.append( FreshnessFinding( "stale_model", f"{suite}: {other} of {len(cases)} cases recorded on another model " - f"than {source_model}; evalshift capture sync replaces them as new " - "captures arrive", + f"than {source_model}; {advice}", ) ) else: @@ -128,11 +133,17 @@ def suite_freshness( ) ) if max_cases is not None and len(cases) > max_cases: + reviewed = sum(1 for c in cases if is_reviewed(c.example)) + # Reviewed cases never leave, so a trim stops at them. + trim = ( + f"evalshift capture sync trims it only down to its {reviewed} reviewed case(s)" + if reviewed > max_cases + else "evalshift capture sync trims it" + ) findings.append( FreshnessFinding( "over_cap", - f"{suite}: {len(cases)} cases, over captures.max_cases {max_cases}; " - "evalshift capture sync trims it", + f"{suite}: {len(cases)} cases, over captures.max_cases {max_cases}; {trim}", ) ) return SuiteFreshness( diff --git a/src/evalshift_cli/cli/commands/capture.py b/src/evalshift_cli/cli/commands/capture.py index fe6b666..97fa92b 100644 --- a/src/evalshift_cli/cli/commands/capture.py +++ b/src/evalshift_cli/cli/commands/capture.py @@ -1348,7 +1348,14 @@ def capture_sync( offline: _OfflineOption = False, base: _BaseOption = None, ) -> None: - """Promote every capture into a suite and wire the suites: block into evalshift.yaml.""" + """Promote captures into suites and wire the suites: block into evalshift.yaml. + + Without limits, every capture is promoted. With captures.max_cases set, + each suite keeps at most that many cases, swapping stale ones (older than + captures.max_age, or recorded on another model than defaults.source_model) + for fresh captures; reviewed cases are never retired. captures.max_age + keeps older captures out of a suite. + """ console = Console() _check_rounds(rounds) limits = _sync_limits(config_path, console=console, warn=True) diff --git a/src/evalshift_cli/cli/commands/doctor.py b/src/evalshift_cli/cli/commands/doctor.py index 2dbef51..9c05f83 100644 --- a/src/evalshift_cli/cli/commands/doctor.py +++ b/src/evalshift_cli/cli/commands/doctor.py @@ -70,7 +70,7 @@ # Row name for the judge / embedding key check (one row per keyless model). EVALUATOR_KEYS_CHECK: Final = "evaluator keys" # Row names for the bounded-suite checks: one ``suite freshness`` row per captured suite (or -# per finding), and a ``captures limits`` nudge when a bucket is set without limits. +# per finding), and a ``captures limits`` nudge when a store is set without max_cases. SUITE_FRESHNESS_CHECK: Final = "suite freshness" CAPTURES_LIMITS_CHECK: Final = "captures limits" # The capture SDK: a declared dependency of this package, and the owner of the diff --git a/tests/unit/test_captures_freshness.py b/tests/unit/test_captures_freshness.py index fbbca26..a6a24d3 100644 --- a/tests/unit/test_captures_freshness.py +++ b/tests/unit/test_captures_freshness.py @@ -9,7 +9,7 @@ from evalshift_cli.captures.freshness import SuiteFreshness, suite_freshness from evalshift_cli.captures.models import PromotedCase from evalshift_cli.captures.promote import write_promoted_case -from evalshift_cli.suite.models import SuiteExample +from evalshift_cli.suite.models import ExpectedToolCall, SuiteExample NOW = datetime(2026, 10, 16, 12, tzinfo=UTC) @@ -77,8 +77,20 @@ def test_cases_on_another_model_are_counted(tmp_path: Path) -> None: _write(tmp_path, 1, days=1, model="claude-sonnet-4-5") [finding] = _check(tmp_path).findings assert finding.kind == "stale_model" - assert finding.message.startswith( - "alpha: 1 of 2 cases recorded on another model than claude-sonnet-5" + assert finding.message == ( + "alpha: 1 of 2 cases recorded on another model than claude-sonnet-5; " + "evalshift capture sync replaces them as new captures arrive" + ) + + +def test_without_max_cases_nothing_replaces_cases_on_another_model(tmp_path: Path) -> None: + # Sync only retires a case to make room under max_cases; max_age alone never replaces one. + _write(tmp_path, 0, days=1) + _write(tmp_path, 1, days=1, model="claude-sonnet-4-5") + [finding] = _check(tmp_path, max_cases=None).findings + assert finding.message == ( + "alpha: 1 of 2 cases recorded on another model than claude-sonnet-5; " + "set captures.max_cases for evalshift capture sync to replace them" ) @@ -98,6 +110,32 @@ def test_a_suite_over_its_cap_is_reported(tmp_path: Path) -> None: ) +def test_an_over_cap_suite_is_trimmed_only_down_to_its_reviewed_cases(tmp_path: Path) -> None: + reviewed = [ExpectedToolCall(tool_name="lookup", provenance="reviewed")] + for i in range(4): + write_promoted_case( + PromotedCase( + name=f"cap_{i}", + suite="alpha", + from_capture=f"cap_{i}", + captured_at=(NOW - timedelta(days=1)).isoformat(), + source_model="claude-sonnet-5", + example=SuiteExample( + id=f"cap_{i}", + inputs={"input": f"q{i}"}, + toolset_ref="sha256:" + "cd" * 32, + expected_tools=reviewed if i < 3 else None, + ), + ), + base=tmp_path, + ) + [finding] = _check(tmp_path, max_cases=2).findings + assert finding.message == ( + "alpha: 4 cases, over captures.max_cases 2; " + "evalshift capture sync trims it only down to its 3 reviewed case(s)" + ) + + def test_an_unreadable_case_file_is_a_finding_not_a_crash(tmp_path: Path) -> None: _write(tmp_path, 0, days=1) (tmp_path / "suites" / "alpha" / "broken.json").write_text("{", encoding="utf-8") diff --git a/tests/unit/test_cli_capture_limits.py b/tests/unit/test_cli_capture_limits.py index 51a23f8..b383a5a 100644 --- a/tests/unit/test_cli_capture_limits.py +++ b/tests/unit/test_cli_capture_limits.py @@ -502,6 +502,14 @@ def test_force_replaces_a_stale_twin_whatever_the_capture_order( assert _cases(tmp_path) == {"b_new"} +def test_help_describes_bounded_suites() -> None: + result = runner.invoke(app, ["capture", "sync", "--help"]) + flat = " ".join(result.stdout.split()) + assert "Promote every capture" not in flat + assert "captures.max_age" in flat + assert "captures.max_cases" in flat + + def _mark_reviewed(path: Path) -> None: data = json.loads(path.read_text(encoding="utf-8")) data["example"]["expected_tools"] = [{"tool_name": "lookup", "provenance": "reviewed"}] From 9d401681d68beeb4eaf2dfd176db9f9d45e72fc1 Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:46:02 +0200 Subject: [PATCH 25/26] docs: where to mark a captured case reviewed, and this wave's sync rules - Captured suites take provenance: reviewed in the case file (.evalshift/suites//.json), not golden.jsonl, which sync regenerates; hand-written suites keep the golden-row guidance. - captures.max_age is at most 36500d; omit it for no age limit. - Retirement targets the case file; a failed delete is reported; a suite synced only from its directory is not wired back in; dedup keeps the claim of every case that stays. - Fetch window: a conversation straddling it loses earlier turns on a fresh checkout; pass the full messages list to capture.model_call. - Freshness messages quote the conditional wording. - CHANGELOG: the first sync writes captured_at / source_model even without limits, so everyone running capture commands upgrades together; a suite never shrinks because its cases went stale. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 27 ++++++++++++++-------- DOCS.md | 8 +++---- docs/agents.md | 12 ++++++++-- docs/configuration.md | 52 +++++++++++++++++++++++++++++++++++-------- llms-full.txt | 48 +++++++++++++++++++++++++++------------ 5 files changed, 109 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7784ae..efa1f09 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,15 +24,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 captures as they arrive. A stale case leaves at once when a fresh capture of the same question is added in its place; if its fresh copy is not added, the stale case leaves only to make room, like any other stale case. So a - suite that has nothing to replace its stale cases with never shrinks. A - conversation enters whole or not at all. Cases a human marked - `provenance: reviewed` are never retired. The same captures give the same - suite on any machine, and syncing again with nothing new changes no file. + suite never shrinks because its cases went stale (an over-cap suite is still + trimmed to `max_cases`). A conversation enters whole or not at all. Cases a + human marked `provenance: reviewed` are never retired; in a captured suite + the mark goes in the case file (`.evalshift/suites//.json`), + not in `golden.jsonl`, which sync regenerates. Sync knows a case by its + file, so a renamed or copied case file is retired (or kept) on its own. + The same captures give the same suite on any machine, and syncing again + with nothing new changes no file. `max_cases` also trims suites sync did not build: suites promoted by hand with `capture promote`, and suites whose entry is `managed: false` (that flag freezes the YAML entry, not the case files). A suite whose case files cannot be read is left untouched, with a warning, and the other suites - still sync. + still sync. A suite planned only because its directory holds case files + is wired into `suites:` only if it already has an entry there. - **Guardrail.** If none of a suite's cases and captures was recorded on `defaults.source_model`, sync warns (`none of N case(s) and capture(s) were recorded on …; not filtering by model — check @@ -69,10 +74,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 limits. When the file cannot be read and has a top-level `captures:` key, sync says so (`could not read (); syncing without captures.max_age / captures.max_cases.`), online and offline. -- Older CLIs reject the new `captures` keys and the new case-file fields: bump - the CLI everywhere `capture` commands run (and a pinned CI action's - `evalshift-version`) before adding them. CI's `compare` reads only - `golden.jsonl`, whose format is unchanged. A run left in progress by an +- The first `capture sync` on this CLI writes `captured_at` and + `source_model` into case files, even without the limits, and older CLIs + reject case files carrying them (and the new `captures` keys). Everyone + who runs `capture` commands, teammates and CI alike, upgrades together: + bump the CLI (and a pinned CI action's `evalshift-version`) everywhere + before the first sync. `captures.max_age` accepts at most `36500d`; omit + it for no age limit. CI's `compare` reads only `golden.jsonl`, whose + format is unchanged. A run left in progress by an older CLI cannot be `--resume`d across the upgrade (config-hash mismatch); start a fresh one. diff --git a/DOCS.md b/DOCS.md index 736b19d..6b906d2 100644 --- a/DOCS.md +++ b/DOCS.md @@ -177,7 +177,7 @@ captures: and `capture sync` and `capture list` fetch what is new before they read the local directory — captures are immutable and toolsets content-addressed, so only objects missing locally are downloaded, captures already promoted are skipped (so `capture clean` never causes a re-download; `capture sync --repair` alone re-fetches the promoted captures its rebuilds need, past `--since` too), and each file is written atomically. Bucket keys are untrusted: one not shaped like the SDK's layout — a suite directory containing `:`, `/`, `\`, NUL or `..`, or starting or ending with `.` or a space, included — is counted as malformed and never written (a key with a trailing newline included), and no key can place a file outside `.evalshift/`; folder-marker keys ending in `/` (the S3 console's "Create folder", ADLS directories) are skipped without being counted. One summary line reports it (`fetched 12 capture(s) and 1 toolset(s) from s3://…`; with `capture list --json` it goes to stderr, so stdout stays pure JSON). `--since 24h` limits a fetch to recently written captures, `--offline` skips it, and `capture fetch` does only that step. `promote`, `diff` and `clean` stay local. Install the matching client library — `pip install boto3`, `google-cloud-storage` or `azure-storage-blob azure-identity` — and authenticate with the provider's normal chain (an OIDC role in CI, `aws sso login` / `gcloud auth application-default login` / `az login` locally); credentials never go in the URI, and a value containing `@` or `?` fails to load; no rejected value is echoed back, at most its scheme. A missing client library exits 1 naming the `pip install ` to run; a list or download failure exits 1, so `sync` promotes nothing from a partial fetch, with a hint naming the provider's credential chain (`aws sso login` / `AWS_PROFILE` / the CI OIDC role, `gcloud auth application-default login`, `az login`); a local write failure names the file and asks you to check free disk space and permissions instead. If `evalshift.yaml` cannot be read or does not validate, `sync` and `list` skip the fetch and work on the local directory as before, warning only when the file has a top-level `captures:` key. Full field reference: [`captures`](docs/configuration.md#captures). -**Bounded suites.** Together, `captures.max_age` (a duration: `30m`, `24h`, `14d`) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) is retired outright when a fresh capture of the same question is added in its place; if its fresh copy is not added, the stale case leaves only to make room, like any other stale case. Stale cases that leave to make room go other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). +**Bounded suites.** Together, `captures.max_age` (a duration: `30m`, `24h`, `14d`, at most `36500d`; omit it for no age limit) and `captures.max_cases` (an int ≥ 1), both optional, keep each captured suite a capped, route-balanced sample of recent production on `defaults.source_model`; `evalshift init` scaffolds `max_age: 14d` and `max_cases: 200`. Setting either one turns the feature on: `max_age` filters what enters a suite and marks old cases stale, and `max_cases` caps and route-balances each suite. Without them `capture sync` promotes and keeps exactly what it did before, and `defaults.source_model` alone changes nothing. Case files record `captured_at` and `source_model` (the first `model_call`'s `model_id`) either way, and sync backfills both on existing cases whose capture is still local (`backfilled capture time and model on N case(s)`). With the limits set, sync promotes only captures recorded within `max_age` on `defaults.source_model`. Models match on bare names: lower-cased, provider prefix and one trailing snapshot date (`-20260301`, `-2026-08-07`) dropped, with no alias lookup, so set `defaults.source_model` to the id your app sends. If none of a suite's cases and captures was recorded on it, sync warns (`alpha: none of 212 case(s) and capture(s) were recorded on claude-sonnet-5-5 (seen: …); not filtering by model — check defaults.source_model`) and does not filter that suite by model, so a typo or a rollout whose first new-model capture has not arrived can neither empty nor freeze a suite. `max_cases` caps every suite under `.evalshift/suites/`, including ones promoted by hand with `capture promote` and ones whose entry is `managed: false` (that flag freezes the YAML entry, not the case files): new conversations enter whole or not at all, least-represented tool route first, in a fixed hash order, so the same captures give the same suite on any machine. A stale case (too old, or recorded on another model) is retired outright when a fresh capture of the same question is added in its place; if its fresh copy is not added, the stale case leaves only to make room, like any other stale case. Stale cases that leave to make room go other-model first, then oldest, so a suite never shrinks because it went stale. A suite still over the cap loses fresh cases from its most-represented route, oldest first. Cases marked `provenance: reviewed` are never retired; in a captured suite, mark them in the case file `.evalshift/suites//.json` (on the calls under `example.expected_tools`, and `example.expected_tool_rounds` when present), not in `golden.jsonl`, which sync regenerates from the case files on every run. `max_age` also bounds `capture sync`'s automatic fetch (`skipped 11 older than captures.max_age (14d)`); `--since` overrides it, and `capture list` and `capture fetch` do not apply it. On a fresh checkout or after `capture clean`, a conversation that straddles the window loses its earlier turns to it, and a turn that does not record its own messages list then rebuilds a truncated history: pass the full messages list to `capture.model_call`. `doctor` and `compare` report stale suites (below). Expire the bucket's `captures/` prefix with a lifecycle rule after a period ≥ `max_age`; the CLI never deletes from the bucket. Full rules, output and lifecycle examples: [Bounded suites](docs/configuration.md#bounded-suites). **`capture sync`** is the workhorse. Per suite it: @@ -189,7 +189,7 @@ and `capture sync` and `capture list` fetch what is new before they read the loc 6. Skips captures whose replayed content duplicates an already-promoted case or an earlier capture in the same run — duplicate examples inflate *n* and corrupt the paired statistics (`--keep-duplicates` opts out). Dedup is seeded from the cases already in the suite dir, so it holds across repeated syncs. 7. Skips captures whose turn recorded an `error` event — a turn that died before the agent acted is not ground truth, and promoting it would assert `expected_no_tools: true` on a question that needed a tool. `--allow-errored` promotes it anyway (still never asserting `expected_no_tools`). `capture promote` exits non-zero on the same condition. Separately and unconditionally — `--allow-errored` does not help — a capture whose first `model_call` has no `toolset_ref` is refused: the SDK did not record what tools were offered, so there is nothing to carry, and re-capturing with a current `evalshift-sdk` is the only fix. Also unconditionally, a capture whose recorded messages carry content a text replay can't carry — an image, a document, audio, a server-tool result, an unknown block type — is skipped and counted on its own (`capture promote` refuses it): a replay without it is not ground truth. Last, as a safety net, a capture whose case fails the suite schema's own validation in a way none of these rules anticipates is skipped with pydantic's first error and counted on its own (`skipped N capture(s) that could not form a valid case`; `capture promote` refuses it) instead of aborting the sync; hand-write that case or report the capture. 8. Warns when two captures claim the same `(conversation_id, turn_index)` (a retried turn), and when a promoted turn contains a failed tool result (`error`, or `{"success": false}`). Both stay warnings — see [Agent evals → What does not belong in a golden suite](docs/agents.md). -9. With `captures.max_age` / `captures.max_cases` set: filters, plans, retires. Captures recorded before now − `max_age`, or on another model than `defaults.source_model`, are not eligible. With `max_cases` set, sync plans each suite (every suite directory with case files, not only those with new captures; `--suite` narrows it). It fills the free slots route-balanced, a conversation at a time. It deletes the case files of the stale cases it retires and of any fresh cases it trims, and rebuilds `golden.jsonl` from what is left; reviewed cases stay. A suite whose case files cannot be read is left unchanged, with a warning (`alpha: could not read its promoted cases (ValidationError); left it unchanged.`). After the summary line, sync prints one line per suite: ` planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model` (zero counts left out; `max_age` alone prints only ` planner: not eligible: …`). Syncing again with nothing new changes no file. If `evalshift.yaml` cannot be read, sync runs without the limits and, when the file has a top-level `captures:` key, says so: `could not read evalshift.yaml (); syncing without captures.max_age / captures.max_cases.` Sync reads the file under `--offline` too. +9. With `captures.max_age` / `captures.max_cases` set: filters, plans, retires. Captures recorded before now − `max_age`, or on another model than `defaults.source_model`, are not eligible. With `max_cases` set, sync plans each suite (every suite directory with case files, not only those with new captures; `--suite` narrows it). It fills the free slots route-balanced, a conversation at a time. It deletes the case files of the stale cases it retires and of any fresh cases it trims, and rebuilds `golden.jsonl` from what is left; reviewed cases stay. A case is known by its file, not the `name` inside it, so a renamed or copied case file is planned and deleted on its own; a deletion that fails is reported (`alpha: could not delete cap_0.json; not counted as retired.`) and not counted. A suite planned only because its directory holds case files is wired into `suites:` only if it already has an entry there. A suite whose case files cannot be read is left unchanged, with a warning (`alpha: could not read its promoted cases (ValidationError); left it unchanged.`). After the summary line, sync prints one line per suite: ` planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model` (zero counts left out; `max_age` alone prints only ` planner: not eligible: …`). Syncing again with nothing new changes no file. If `evalshift.yaml` cannot be read, sync runs without the limits and, when the file has a top-level `captures:` key, says so: `could not read evalshift.yaml (); syncing without captures.max_age / captures.max_cases.` Sync reads the file under `--offline` too. 10. Writes `.evalshift/suites//golden.jsonl` and rewrites the managed `suites:` block in `evalshift.yaml` (between the `>>> evalshift suites` markers). 11. After the write (or after printing the block for you to paste), checks the CI pin: if a workflow under `.github/workflows/` uses `evalshift/evalshift-action` with an `evalshift-version` older than this CLI, with no pin at all, or with pins that are all newer than this CLI, it prints a warning naming the workflow and job plus the fix — the exact `evalshift-version: ""` line to set, or, for a newer pin, `pip install -U evalshift` locally. Advisory only — sync never edits a workflow and the exit code is unchanged. See [Pin drift](#pin-drift). @@ -468,7 +468,7 @@ Each entry in `expected_tools`: - `arguments: null` → name-only check. - `match_strategy`: `exact` (arguments must match exactly), `subset` (default; expected keys must be present and equal, extras allowed), `contains_per_field` (per-field containment). -- `provenance`: `captured` (default, what `capture promote`/`sync` write — transcribed from the source model's own call, unverified) or `reviewed` (a human has confirmed it). Scoring is identical; the flag only decides whether the run discloses that its ground truth is source-derived — see [Agent evaluation → Ground truth](#ground-truth). +- `provenance`: `captured` (default, what `capture promote`/`sync` write — transcribed from the source model's own call, unverified) or `reviewed` (a human has confirmed it; in a captured suite, set it in the case file, since `capture sync` regenerates `golden.jsonl`). Scoring is identical; the flag only decides whether the run discloses that its ground truth is source-derived — see [Agent evaluation → Ground truth](#ground-truth). Validators enforced at load: exactly one of `toolset_ref` / `tools` is required — neither, or both, fails to load; `expected_no_tools: true` is incompatible with non-empty `expected_tools`, a non-empty `expected_tool_rounds`, or a nonzero `expected_tool_count` — and so is an empty toolset, spelled `tools: []` or a `toolset_ref` naming the empty toolset (a call offered no tools cannot produce a tool call); `tool_result_fixtures` requires `expected_tool_rounds`, cannot cover more rounds than it has, and every covered round needs one result per expected call with a matching `tool_name`; `history` may contain at most one `system` message and it must come first; duplicate ids across the suite are rejected. The loader collects **all** schema errors before failing, so you fix a broken suite in one pass. @@ -616,7 +616,7 @@ in `evalshift.yaml` reads it. Per-example expectations (`expected_tools`, `expected_no_tools`, `expected_tool_count`, `expected_parallel`) are described under [The golden suite](#the-golden-suite). You rarely write them by hand — `capture promote`/`capture sync` derive them from recorded production behaviour (`--strict-args`, `--names-only`, `--tool-count` control how strict the derived expectations are). -Derived expectations are marked `provenance: captured` (the default), meaning their arguments were transcribed verbatim from the source model's own recorded call — nobody has checked that they are *right*. On such a row the source scores 1.0 **by construction**, so an `against: expected` gate degenerates into what `against: source` already measured: target deviation from source. The run discloses that rather than letting `source_score: 1.0` read as evidence — when every scored row is `captured`, `migration_decision.json` gains a recommendation naming the count and the caveat. Set `provenance: reviewed` on a row once a human has confirmed its arguments; scoring is identical either way, and the disclosure goes silent as soon as one row is reviewed (a blanket disclaimer would understate the rows someone did check). +Derived expectations are marked `provenance: captured` (the default), meaning their arguments were transcribed verbatim from the source model's own recorded call — nobody has checked that they are *right*. On such a row the source scores 1.0 **by construction**, so an `against: expected` gate degenerates into what `against: source` already measured: target deviation from source. The run discloses that rather than letting `source_score: 1.0` read as evidence — when every scored row is `captured`, `migration_decision.json` gains a recommendation naming the count and the caveat. Set `provenance: reviewed` once a human has confirmed a row's arguments: on the golden row in a hand-written suite, and in a captured suite in the case file `.evalshift/suites//.json` (on the calls under `example.expected_tools`, and `example.expected_tool_rounds` when present), because `capture sync` regenerates `golden.jsonl` from the case files on every run. Scoring is identical either way, and the disclosure goes silent as soon as one row is reviewed (a blanket disclaimer would understate the rows someone did check). ### Trace invariants diff --git a/docs/agents.md b/docs/agents.md index ab2d6ec..ac23a22 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -560,13 +560,21 @@ source was right. Nothing is wrong with the number, so the run says so instead o changing it: when every scored row is `captured`, `migration_decision.json` carries a recommendation naming the count and the caveat. -Set `provenance: reviewed` on a golden row once a human has checked its -arguments: +Set `provenance: reviewed` once a human has checked a row's arguments. In a +hand-written suite, set it on the golden row: ```jsonl {"id": "ex_refund_01", "inputs": {"query": "..."}, "toolset_ref": "sha256:1a2b...", "expected_tools": [{"tool_name": "issue_refund", "arguments": {"order_id": "A-1", "amount_usd": 40.0}, "provenance": "reviewed"}]} ``` +In a captured suite, set it in the promoted case file instead, +`.evalshift/suites//.json`, on the calls under +`example.expected_tools` (and under `example.expected_tool_rounds`, when the +case has it), and commit that file. `capture sync` regenerates `golden.jsonl` +from the case files on every run, so a mark made in `golden.jsonl` is gone after +the next sync. With `captures.max_cases` set, sync never retires a reviewed case +(see [Bounded suites](configuration.md#bounded-suites)). + Scoring is identical either way. The disclosure goes silent as soon as one row is `reviewed`: a suite someone has started checking is no longer uniformly source-derived, and a blanket disclaimer over it would understate the rows they diff --git a/docs/configuration.md b/docs/configuration.md index aba22ee..0acce8e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -904,9 +904,10 @@ Each `expected_tools` entry carries `provenance`: `captured` (the default, and what `capture promote` / `capture sync` write) means its arguments were transcribed verbatim from the source model's own recorded call — nobody has checked that they are *right*. Set it to `reviewed` once a human has confirmed -the row. Scoring is identical either way; the flag only controls whether the run -discloses that its `against: expected` ground truth is source-derived (see -[Agent migrations](agents.md#scoring-arguments-against-ground-truth)). +the row (in a captured suite, in the case file; see +[Bounded suites](#bounded-suites)). Scoring is identical either way; the flag +only controls whether the run discloses that its `against: expected` ground +truth is source-derived (see [Agent migrations](agents.md#scoring-arguments-against-ground-truth)). **`toolset_ref` (string) or `tools` (list) — required, exactly one of the two.** Every model call records the toolset it was offered, so every suite @@ -1085,7 +1086,7 @@ captures: | Field | Type | Default | Description | | ----------- | --------------------------------- | ------- | ----------- | | `store` | string (URI) | (none) | Object store the SDK writes to. `capture sync` and `capture list` fetch new captures and toolset sidecars from it into the local directory first; `capture fetch` does only that step. | -| `max_age` | duration (`30m`, `24h`, `14d`; > 0) | (none) | Captures recorded longer ago than this never enter a suite through `capture sync` (`capture promote` ignores the limits), and promoted cases that old are stale: with `max_cases` also set, `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | +| `max_age` | duration (`30m`, `24h`, `14d`; > 0, ≤ `36500d`) | (none) | Captures recorded longer ago than this never enter a suite through `capture sync` (`capture promote` ignores the limits), and promoted cases that old are stale: with `max_cases` also set, `capture sync` replaces them as fresh captures arrive. Also the window `capture sync` fetches from `store`. Omit for no age limit. | | `max_cases` | int (≥ 1) | (none) | The most cases `capture sync` keeps in each captured suite. Omit and suites grow with every sync. | Setting either `max_age` or `max_cases` turns [bounded suites](#bounded-suites) on. Both work @@ -1093,7 +1094,9 @@ without `store`: they govern how captures become suites, whether the captures ca from a bucket. A `max_age` that is not a duration fails to load; `evalshift validate` reports it as `captures.max_age: Value error, expected a duration like 30m, 24h or 14d`, followed for a string by the value it got (`, got '2w'`). A bare number such as `14` is rejected too, without that suffix, -because it would silently mean seconds. +because it would silently mean seconds. So is a `max_age` above 100 years, which some write to mean +"no limit": `captures.max_age: Value error, must be at most 36500d (100 years); omit +captures.max_age for no age limit`. CLI versions before 1.3.0 reject the `captures:` key, and versions before this release reject `max_age` and `max_cases` (the config is strict). Bump the CLI, and a pinned GitHub Action's @@ -1209,17 +1212,41 @@ trimmed. If they alone exceed the cap, sync keeps them and warns (`alpha: 3 reviewed case(s) over captures.max_cases 2; reviewed cases are never retired, so it stays over.`). +In a captured suite, set the mark in the case file, `.evalshift/suites//.json`, on +the calls under `example.expected_tools` (and under `example.expected_tool_rounds`, when the case +has it), and commit that file. Not in `golden.jsonl`: sync regenerates it from the case files on +every run, so a mark made there is gone after the next sync. + +```json +"example": { + "id": "cap_0", + "expected_tools": [ + {"tool_name": "issue_refund", "arguments": {"order_id": "A-1"}, "provenance": "reviewed"} + ] +} +``` + +Sync knows a case by its file, not by the `name` inside it: retiring a case deletes the file it +was read from, so a case file you renamed or copied is planned on its own. A deletion that fails +is reported (`alpha: could not delete cap_0.json; not counted as retired.`) and left out of the +counts. A new capture whose case file name is already taken in the suite is counted as already +promoted and not added. + With `max_cases` set, sync plans every suite directory under `.evalshift/suites/` that holds case files, even with no new captures, so an over-cap suite is trimmed on the next sync. That includes suites sync did not build: suites promoted by hand with `capture promote`, and suites whose `suites:` entry is `managed: false`. `managed: false` freezes the YAML entry, not the case files. -`--suite` narrows the plan to one suite. A suite directory whose case files cannot be read is left -untouched: its `golden.jsonl` and `suites:` entry stay as they are, sync warns -(`alpha: could not read its promoted cases (ValidationError); left it unchanged.`), and the other -suites still sync. +`--suite` narrows the plan to one suite. A suite planned only because its directory holds case +files (no capture of it this run) has its `golden.jsonl` rebuilt, but is wired into `suites:` only +if it already has an entry there: a suite you took out of `suites:` stays out. A suite directory +whose case files cannot be read is left untouched: its `golden.jsonl` and `suites:` entry stay as +they are, sync warns (`alpha: could not read its promoted cases (ValidationError); left it +unchanged.`), and the other suites still sync. Duplicates: with `max_cases` set, a stale case that is not reviewed gives up its claim on its content, so a fresh capture of the same question can replace it, under `--keep-duplicates` too. +A case that stays keeps its claim, so if a suite already holds the same content twice, a fresh +copy is still skipped as a duplicate. With `max_age` alone, nothing is ever retired: too-old and other-model captures are not promoted, and a fresh capture of a stale case's question is skipped as a duplicate. @@ -1249,6 +1276,11 @@ downloaded. `EVALSHIFT_SAMPLE_RATE` on the agent governs the volume. On a fresh committed case files tell the fetch which captures are already promoted, so only new ones inside the window are downloaded. +On a fresh checkout or after `capture clean`, a conversation that straddles the `max_age` window +loses its earlier turns to the fetch window. A later turn that does not record its own messages +list then rebuilds a truncated history from the turns it has. Pass the full messages list to +`capture.model_call` so each turn carries its own history. + If `evalshift.yaml` cannot be read or does not validate, sync runs without the limits. When the file has a top-level `captures:` key, it says so, online or `--offline`: `could not read evalshift.yaml (1 schema problem found); syncing without captures.max_age / @@ -1265,8 +1297,10 @@ measure against `defaults.source_model`, not `--from`. (`alpha: 200 cases, captured 2d–12d ago`), or one `warn` row per finding: - `alpha: newest case captured 23d ago (captures.max_age 14d); refresh with evalshift capture sync` - `alpha: 18 of 200 cases recorded on another model than claude-sonnet-5-5; evalshift capture sync replaces them as new captures arrive` + (without `max_cases`, which is what makes sync replace them: `…; set captures.max_cases for evalshift capture sync to replace them`) - `alpha: no case recorded on claude-sonnet-5-5 (seen: claude-sonnet-5-20260301 ×200); check defaults.source_model, or record new captures` - `alpha: 2940 cases, over captures.max_cases 200; evalshift capture sync trims it` + (when the reviewed cases alone exceed the cap: `…; evalshift capture sync trims it only down to its 230 reviewed case(s)`) - `alpha: 2 case file(s) could not be read; the counts leave them out` - `doctor` also adds a `captures limits` `warn` row when `store` is set and `max_cases` is not: `suites grow with every capture sync; set captures.max_cases and captures.max_age`, or diff --git a/llms-full.txt b/llms-full.txt index 20947e4..f158074 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -208,10 +208,13 @@ evalshift doctor cases, captured 2d–12d ago"; else one warn (exit 0, never fail) per finding: "alpha: newest case captured 23d ago (captures.max_age 14d); refresh with evalshift capture sync" "alpha: 18 of 200 cases recorded on another model than ; evalshift - capture sync replaces them as new captures arrive" + capture sync replaces them as new captures arrive" (without max_cases: "...; set + captures.max_cases for evalshift capture sync to replace them") "alpha: no case recorded on (seen: ×N); check defaults.source_model, or record new captures" (the guardrail case: nothing matches) - "alpha: 2940 cases, over captures.max_cases 200; evalshift capture sync trims it" + "alpha: 2940 cases, over captures.max_cases 200; evalshift capture sync trims it" (reviewed + cases alone over the cap: "...; evalshift capture sync trims it only down to its N reviewed + case(s)") "alpha: N case file(s) could not be read; the counts leave them out" Measured against defaults.source_model, models matched on bare names (see capture sync). evalshift run [-f/--from MODEL] [-t/--to MODEL] [-c CONFIG] [-s/--suite FILE] @@ -430,8 +433,8 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri default (duplicates inflate n, corrupt paired stats); --keep-duplicates opts out. Dedup is seeded from cases already promoted in the suite dir, so it spans sync runs. Reports "wired generation config for N case(s)" when promoted captures carried one. - BOUNDED SUITES (captures.max_age: duration 30m|24h|14d > 0; captures.max_cases: int >= 1; both - default unset). OPT-IN: every rule below applies only when at least one is set; + BOUNDED SUITES (captures.max_age: duration 30m|24h|14d, > 0, <= 36500d; captures.max_cases: + int >= 1; both default unset). OPT-IN: every rule below applies only when at least one is set; defaults.source_model alone changes nothing; without them sync promotes/keeps exactly what it did before. ALWAYS (limits or not): PromotedCase gains captured_at (envelope created_at, verbatim) + source_model (first model_call's model_id, the call a replay re-sends); sync @@ -461,8 +464,17 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri new captures; includes suites promoted by hand with `capture promote` and suites whose entry is managed: false -- that flag freezes the YAML entry, not the case files); --suite narrows. Kept cases: reviewed (any expected tool call provenance: reviewed -> NEVER retired), fresh, - stale. Unit = conversation group (conversation_id; a lone capture is its own group), added - whole or skipped (next group tried). slots = max_cases - reviewed - fresh; repeatedly add the + stale. Captured suites: the reviewed mark goes IN THE CASE FILE + /suites//.json (example.expected_tools[], and + example.expected_tool_rounds when present), NEVER golden.jsonl: sync regenerates golden.jsonl + from the case files every run, so a mark there is lost. A case is its FILE, not its "name" + field: planned by file stem, a retirement deletes the file the case was read from (a renamed + or copied case file is planned on its own); a failed delete prints ": could not + delete ; not counted as retired." and is not counted. A capture whose case file name + is already taken is counted already-promoted and never planned; a planned write that fails + is not counted as added, and sync re-plans so its stale twin stays. Unit = conversation group + (conversation_id; a lone capture is its own group), added whole or skipped (next group + tried). slots = max_cases - reviewed - fresh; repeatedly add the next group of the route (sorted round-1 expected tool names; none is its own route) with the fewest cases so far that still has a group that fits (ties by route name); within a route, sha256(group key) order. Retire: a stale case whose content key an added turn replays is @@ -474,11 +486,13 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri (unknown time = oldest). Reviewed over the cap -> kept + ": N reviewed case(s) over captures.max_cases M; reviewed cases are never retired, so it stays over." Dedup seed: a stale non-reviewed case gives up its content claim (max_cases set only), so a fresh re-capture of - its question can replace it. Same captures -> same suite on any machine; re-running with - nothing new changes no file. Retired case files are deleted, then golden.jsonl is rebuilt. A - suite whose case files cannot be read is LEFT UNTOUCHED (golden.jsonl and suites entry as they - were): ": could not read its promoted cases (); left it unchanged." -- other - suites still sync. + its question can replace it; every case that stays keeps its claim (a suite already holding + the content twice still skips the fresh copy). A suite planned only from its directory (no + capture this run) is rebuilt but wired into suites: only if it already has an entry. Same + captures -> same suite on any machine; re-running with nothing new changes no file. Retired + case files are deleted, then golden.jsonl is rebuilt. A suite whose case files cannot be read + is LEFT UNTOUCHED (golden.jsonl and suites entry as they were): ": could not read its + promoted cases (); left it unchanged." -- other suites still sync. Output, after the summary line, one line per planned suite (zero counts left out): " planner: 200 cases — kept 150, retired 50 (32 older than 14d, 18 on another model), added 50 of 2940 eligible; not eligible: 812 older than 14d, 40 on another model" @@ -490,7 +504,10 @@ evalshift capture sync [--suite S] [--input-var NAME=input] [--tag T]... [--stri capture fetch do not apply it. No sampling at fetch time (a listing has key/time/size only; history rebuild needs sibling turns) -- volume inside the window is EVALSHIFT_SAMPLE_RATE's job. Committed case files supply the promoted-id skip list, so a fresh checkout downloads only - new captures inside the window. + new captures inside the window. On a fresh checkout or after capture clean, a conversation + straddling the window loses its earlier turns to it; a turn that does not record its own + messages list then rebuilds a TRUNCATED history -- pass the full messages list to + capture.model_call. Route balancing is coverage-weighted, not traffic-weighted: rate budgets are measured over a balanced sample. The CLI never writes to or deletes from the bucket; expire the captures/ prefix (not toolsets/, which committed suites reference) with a lifecycle rule after a period @@ -903,7 +920,8 @@ captures: trailing slash stripped; default none = local disk only. Client libraries are plain packages: pip install boto3 (s3://; AWS_ENDPOINT_URL covers MinIO/R2/B2), google-cloud-storage (gs://), azure-storage-blob azure-identity (az://). Credentials: each provider's default chain only. - max_age: duration|null = null # N + m|h|d (30m, 24h, 14d), > 0; parsed to a timedelta. Bad + max_age: duration|null = null # N + m|h|d (30m, 24h, 14d), > 0, <= 36500d (above -> "must be + # at most 36500d (100 years); omit captures.max_age for no age limit"); a timedelta. Bad # value -> validate prints "captures.max_age: Value error, expected a duration like 30m, # 24h or 14d" (+ ", got '2w'" for a string; a bare number is rejected too: it would # silently mean seconds). Captures older than this never enter a suite through capture @@ -962,6 +980,8 @@ estimated capture cost uses DeepSeek's API price. # model's own recorded call (promote/sync write this); # nobody checked it is RIGHT. Scoring identical either # way; only gates the disclosure recommendation. + # Captured suite: set reviewed in the case file + # (sync regenerates golden.jsonl from case files). "expected_tool_rounds": [[{...}]]|null, # full recorded agent loop, one list per model turn that # emitted tool calls; expected_tools is normally # expected_tool_rounds[0]. null for pre-v0.3 suites / @@ -1577,7 +1597,7 @@ content leaves the machine. content (inputs + history), NOT the envelope input_hash (the SDK salts that with conversation_id and derives it from the agent's bound args). Seeded from already-promoted cases, so dedup spans sync runs. With captures.max_cases set, a stale non-reviewed case does - not seed it (its fresh re-capture replaces it). + not seed it (its fresh re-capture replaces it); every case that stays does. - Bounded capture sync (captures.max_age / captures.max_cases set): a suite never shrinks because it went stale (stale cases leave only to make room or to be replaced by their fresh twin); reviewed cases are never retired; a conversation is added whole or not at all; the From d500bf0118f18b97d4dc454cb9ad93b1d641aa7f Mon Sep 17 00:00:00 2001 From: Lukas Babaliauskas Date: Sat, 17 Oct 2026 00:48:55 +0200 Subject: [PATCH 26/26] docs(examples): replay capture-first with this CLI, without the captures: block A fresh init now scaffolds captures.max_age / max_cases, and the example's demo captures (recorded 2026-09-08 on a placeholder model) are all older than max_age, so following the README promoted nothing. The README now has the reader delete the captures: block init writes, says why, and shows what sync prints with it left in. The config and promoted case files are regenerated by replaying the walkthrough with this CLI (case files gain captured_at / source_model; the config picks up init's max_invariant_violations line it had drifted from). The drift test now also checks the config outside the managed region against init's output minus CAPTURES_SCAFFOLD. Co-Authored-By: Claude Opus 5.5 --- README.md | 3 +- .../cap_17098c7050c54dbc8116c2db015d5707.json | 4 +- .../cap_7ca7d4f3a9e94b73bb8960451c1ba07b.json | 4 +- .../cap_e8d69324b0724073a72f4424cbe95473.json | 4 +- examples/capture-first/README.md | 37 ++++++++++++++++--- examples/capture-first/evalshift.yaml | 1 + tests/unit/test_config_suites.py | 29 ++++++++++++++- 7 files changed, 72 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 3b6790d..5c5847f 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,8 @@ evalshift compare --suite-name support_agent --to See [docs/sdk.md](docs/sdk.md) for the full capture contract, and [`examples/capture-first/`](examples/capture-first/) for those three commands checked in end to end — the instrumented agent, the captures it wrote, the -promoted suite, and the `suites:` block `capture sync` filled in. Can't +promoted suite, and the `suites:` block `capture sync` filled in (its demo +captures are old, so its config drops the `captures:` limits `init` writes). Can't instrument the agent? A hand-written `golden.jsonl` works just as well — see [Getting started](docs/getting-started.md). diff --git a/examples/capture-first/.evalshift/suites/oncall_triage/cap_17098c7050c54dbc8116c2db015d5707.json b/examples/capture-first/.evalshift/suites/oncall_triage/cap_17098c7050c54dbc8116c2db015d5707.json index 01ed465..b2d15a8 100644 --- a/examples/capture-first/.evalshift/suites/oncall_triage/cap_17098c7050c54dbc8116c2db015d5707.json +++ b/examples/capture-first/.evalshift/suites/oncall_triage/cap_17098c7050c54dbc8116c2db015d5707.json @@ -2,12 +2,14 @@ "name": "cap_17098c7050c54dbc8116c2db015d5707", "suite": "oncall_triage", "from_capture": "cap_17098c7050c54dbc8116c2db015d5707", - "promoted_at": "2026-09-09T15:38:57.231003+00:00", + "promoted_at": "2026-10-16T22:47:04.187698+00:00", "source_input_hash": "9c846a5018d2bc893f6ff29f542c3e4ae08a2acf0ede9f56a442f31fe8782d78", "code_version": "", "promotion_source": "executed", "cost_usd": 0.0, "cost_source": null, + "captured_at": "2026-09-08T21:42:24.265284+00:00", + "source_model": "demo/triage-router", "conversation_id": null, "turn_index": null, "example": { diff --git a/examples/capture-first/.evalshift/suites/oncall_triage/cap_7ca7d4f3a9e94b73bb8960451c1ba07b.json b/examples/capture-first/.evalshift/suites/oncall_triage/cap_7ca7d4f3a9e94b73bb8960451c1ba07b.json index e52a08b..29965bf 100644 --- a/examples/capture-first/.evalshift/suites/oncall_triage/cap_7ca7d4f3a9e94b73bb8960451c1ba07b.json +++ b/examples/capture-first/.evalshift/suites/oncall_triage/cap_7ca7d4f3a9e94b73bb8960451c1ba07b.json @@ -2,12 +2,14 @@ "name": "cap_7ca7d4f3a9e94b73bb8960451c1ba07b", "suite": "oncall_triage", "from_capture": "cap_7ca7d4f3a9e94b73bb8960451c1ba07b", - "promoted_at": "2026-09-09T15:38:57.231182+00:00", + "promoted_at": "2026-10-16T22:47:04.187839+00:00", "source_input_hash": "c46a3ad6b99405eed54341baa177112a5cd619336f02cd3c90d0201ee13166e7", "code_version": "", "promotion_source": "executed", "cost_usd": 0.0, "cost_source": null, + "captured_at": "2026-09-08T21:42:24.263986+00:00", + "source_model": "demo/triage-router", "conversation_id": null, "turn_index": null, "example": { diff --git a/examples/capture-first/.evalshift/suites/oncall_triage/cap_e8d69324b0724073a72f4424cbe95473.json b/examples/capture-first/.evalshift/suites/oncall_triage/cap_e8d69324b0724073a72f4424cbe95473.json index 267a323..98ee8cb 100644 --- a/examples/capture-first/.evalshift/suites/oncall_triage/cap_e8d69324b0724073a72f4424cbe95473.json +++ b/examples/capture-first/.evalshift/suites/oncall_triage/cap_e8d69324b0724073a72f4424cbe95473.json @@ -2,12 +2,14 @@ "name": "cap_e8d69324b0724073a72f4424cbe95473", "suite": "oncall_triage", "from_capture": "cap_e8d69324b0724073a72f4424cbe95473", - "promoted_at": "2026-09-09T15:38:57.231268+00:00", + "promoted_at": "2026-10-16T22:47:04.187903+00:00", "source_input_hash": "cd20f30169092fa1cd79968a47baab2520294c29913aa40b545167922c2ac4d8", "code_version": "", "promotion_source": "executed", "cost_usd": 0.0, "cost_source": null, + "captured_at": "2026-09-08T21:42:24.264720+00:00", + "source_model": "demo/triage-router", "conversation_id": null, "turn_index": null, "example": { diff --git a/examples/capture-first/README.md b/examples/capture-first/README.md index e6dccd1..5048c8b 100644 --- a/examples/capture-first/README.md +++ b/examples/capture-first/README.md @@ -16,10 +16,10 @@ or the evaluators — was typed by hand. | Path | Where it came from | | --- | --- | | `agent.py` | Hand-written. A deterministic on-call triage agent instrumented with `@capture.agent`, `@capture.tool` and `record_model_call`. No LLM is called. | -| `evalshift.yaml` | `evalshift init --provider gemini --no-wire-agents`, then `evalshift capture sync`. Unedited — the only difference from a fresh `init` is the filled-in managed region at the bottom. | +| `evalshift.yaml` | `evalshift init --provider gemini --no-wire-agents`, its `captures:` block deleted (see step 2), then `evalshift capture sync`. Otherwise unedited — the only other difference from a fresh `init` is the filled-in managed region at the bottom. | | `.evalshift/captures/oncall_triage/cap_*.json` | Written by the SDK when `agent.py` ran with `EVALSHIFT_CAPTURE=1`. Three alerts, three captures. | | `.evalshift/toolsets/56503f6d….json` | Written by the SDK. One content-addressed sidecar holding the four tool schemas the router model was offered; every capture and every promoted case points at it by hash. | -| `.evalshift/suites/oncall_triage/cap_*.json` | Written by `evalshift capture sync`. One auditable promoted case per capture, carrying its provenance. | +| `.evalshift/suites/oncall_triage/cap_*.json` | Written by `evalshift capture sync`. One auditable promoted case per capture, carrying its provenance: the capture it came from, when it was recorded (`captured_at`) and on which model (`source_model`). | | `.evalshift/suites/oncall_triage/golden.jsonl` | Written by `evalshift capture sync`, regenerated from the case files above. This is what `run` reads. | | `.gitignore` | Hand-written, and explained below. | @@ -83,6 +83,31 @@ The prompt is the whole reason `prompts:` is still required in a capture-first project: a promoted case carries the *rendered* prompt, so replaying it means echoing that string back verbatim. +`init` also writes a `captures:` block, just above that region: + +```yaml +captures: + max_age: 14d + max_cases: 200 + # store: s3:/// +``` + +**Delete it here**, with the comment above it. The block keeps each captured +suite a capped sample of recent captures recorded on `defaults.source_model`, +but the captures checked in here are demo recordings: they are older than +`max_age`, and were recorded on a placeholder model id (`demo/triage-router`), +not on `defaults.source_model`. With the block in place, `sync` promotes none of +them: + +``` +⚠ oncall_triage: none of 3 case(s) and capture(s) were recorded on gemini-3.1-flash-lite-preview (seen: demo/triage-router ×3); not filtering by model — check defaults.source_model +✓ promoted 0 capture(s) into 0 suite(s). + oncall_triage: 0 cases — kept 0, retired 0, added 0 of 0 eligible; not eligible: 3 older than 14d +``` + +A real project, recording fresh captures on the model it runs, keeps the block — +see [Bounded suites](../../docs/configuration.md#bounded-suites). + ### 3. Promote the captures ```bash @@ -95,8 +120,8 @@ evalshift capture sync run: evalshift compare --suite-name oncall_triage --to ``` -`sync` promotes every capture, rebuilds `golden.jsonl`, and rewrites the managed -region with an entry per suite: +With no `captures:` limits, `sync` promotes every capture, rebuilds +`golden.jsonl`, and rewrites the managed region with an entry per suite: ```yaml # >>> evalshift suites (managed by `evalshift capture sync`) >>> @@ -193,7 +218,9 @@ test run`, and results stay flagged uncertain below n=20. A suite this small demonstrates the wiring, not a verdict. A real capture-first suite grows by leaving the SDK instrumentation in place and re-running `capture sync`; sync de-duplicates on replayed content, seeded from the cases already on disk, so -repeated syncs add cases without inflating *n*. +repeated syncs add cases without inflating *n*. With the `captures:` limits +`init` scaffolds, each sync also keeps the suite a capped, fresh sample instead +of letting it grow without end. ## Why `.evalshift/` is committed here diff --git a/examples/capture-first/evalshift.yaml b/examples/capture-first/evalshift.yaml index f57f838..fb60017 100644 --- a/examples/capture-first/evalshift.yaml +++ b/examples/capture-first/evalshift.yaml @@ -75,6 +75,7 @@ migration_policy: min_equivalence_rate: 0.75 max_tool_argument_drift: 0.20 max_tool_divergence: 0.20 + max_invariant_violations: 0 max_cost_increase: 0.30 max_latency_increase: 0.30 diff --git a/tests/unit/test_config_suites.py b/tests/unit/test_config_suites.py index 27f86f6..a9f89c9 100644 --- a/tests/unit/test_config_suites.py +++ b/tests/unit/test_config_suites.py @@ -14,7 +14,12 @@ from evalshift_cli.captures.promote import PromoteOptions, build_example_from_capture from evalshift_cli.captures.reader import iter_captures, toolset_path -from evalshift_cli.cli.commands._suites import resolve_suite_path +from evalshift_cli.cli.commands._suites import ( + SUITES_MARKER_BEGIN, + SUITES_MARKER_END, + resolve_suite_path, +) +from evalshift_cli.cli.commands.init import CAPTURES_SCAFFOLD, render_minimal_config from evalshift_cli.config.loader import load_config from evalshift_cli.config.models import ( EvalShiftConfig, @@ -235,6 +240,28 @@ def test_config_wires_the_promoted_suite(self) -> None: ) assert resolved.is_file(), f"{resolved} is not committed" + def test_config_is_a_fresh_init_without_its_captures_block(self) -> None: + """Outside the managed region, the config is what the README's ``init`` writes. + + ``init --provider gemini --no-wire-agents`` (profile ``model-upgrade``), minus the + ``captures:`` block the README has the reader delete: the demo captures are older + than ``max_age`` and recorded on a placeholder model, so a bounded sync would + promote none of them. + """ + + def outside_region(text: str) -> str: + head, _, rest = text.partition(SUITES_MARKER_BEGIN) + _, _, tail = rest.partition(SUITES_MARKER_END) + return head + tail + + committed = (_CAPTURE_FIRST / "evalshift.yaml").read_text(encoding="utf-8") + assert SUITES_MARKER_BEGIN in committed + assert SUITES_MARKER_END in committed + fresh = render_minimal_config(profile="model-upgrade", provider="gemini", env={}) + expected = fresh.replace(CAPTURES_SCAFFOLD + "\n", "") + assert expected != fresh + assert outside_region(committed) == outside_region(expected) + def test_derived_evaluators_are_wired_for_the_suite(self) -> None: """The captures call tools, so sync must have derived both tool families.""" cfg = load_config(_CAPTURE_FIRST / "evalshift.yaml")