From 7ff042f135ac5b22acf6bb4f0ad27817a25ea281 Mon Sep 17 00:00:00 2001
From: Lukas Babaliauskas <l.babaliauskas@gmail.com>
Date: Tue, 13 Oct 2026 22:39:58 +0200
Subject: [PATCH 1/2] fix(policy): count failure categories per example, not
 per row

tool_selection stamps TOOL_SELECTION_DRIFT on both its conformance and its
divergence row, so one example that picked another tool read "Different
tools chosen (2 examples)" in the narrative and the report's top causes.
Count each label once per distinct (prompt, example), as
INVARIANT_VIOLATION already was.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 CHANGELOG.md                         |  8 ++++++++
 DOCS.md                              |  2 ++
 llms-full.txt                        |  2 ++
 src/evalshift_cli/analysis/policy.py | 12 +++++++++---
 tests/unit/test_policy.py            | 21 +++++++++++++++++++++
 5 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3766589..c02349c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Fixed
+
+- Failure categories count examples, not scored rows. `tool_selection` stamps
+  `TOOL_SELECTION_DRIFT` on both its conformance and its divergence row, so one example
+  that picked another tool read "Different tools chosen (2 examples)" in the narrative
+  and 2 in the report's top regression causes. Each label is now counted once per
+  distinct example, as `INVARIANT_VIOLATION` already was.
+
 ## [1.4.2] - 2026-10-12
 
 ### Fixed
diff --git a/DOCS.md b/DOCS.md
index d0d9f41..73f61fe 100644
--- a/DOCS.md
+++ b/DOCS.md
@@ -533,6 +533,8 @@ Pairwise A/B comparison per criterion: the judge sees the two outputs anonymised
 
 Regressions carry machine-readable labels that the report and hosted diff group by: `FORMAT_FAILURE`, `SEMANTIC_REGRESSION`, `TOOL_SELECTION_DRIFT`, `TOOL_GROUND_TRUTH_MISS` (both models missed the same recorded tool ground truth — a broken-harness signal, not a migration finding; see [Migration policy and CI gating](#migration-policy-and-ci-gating)), `ARGUMENT_VALUE_DRIFT`, `TOOL_TRACE_STRUCTURE_DRIFT`, `TOOL_ORDER_DRIFT`, `DANGEROUS_ACTION_DRIFT`, `MISSING_VERIFICATION_STEP`, `UNNECESSARY_TOOL_CALL`, `REFUSAL_REGRESSION`, and `INVARIANT_VIOLATION` (the target broke a hand-written [trace rule](#trace-invariants), stamped whatever the source did). That is the complete set — every label the evaluators emit is declared in `evaluators/failures.py`. The machine labels live in `scores.jsonl`, `report.json` and the bundle; every rendered surface (the HTML report, decision prose, the run narrative) shows the plain-language display name instead — `TOOL_SELECTION_DRIFT` renders as "Different tools chosen" — with the mapping declared beside the labels in `evaluators/failures.py`.
 
+The decision's `failure_categories` (the report's **Top regression causes** panel, and the narrative's "Most common failure … (N examples)") count each label once per **distinct example**, not per scored row: `tool_selection` stamps `TOOL_SELECTION_DRIFT` on both its conformance and its divergence row, so one example that picked another tool is one, not two. `INVARIANT_VIOLATION` is further narrowed to blocking entries — see [Trace invariants](#trace-invariants).
+
 `ARGUMENT_VALUE_DRIFT` counts **regressions**: it is stamped only when the target scored below the source. Under `against: expected` both models can miss the same recorded expectation by the same margin — a zero delta, and a fact about your ground truth rather than a migration defect, already reported as such. Policy budgets are unaffected by the label: `max_tool_argument_drift` counts calls whose *target* score fell below `tool_argument_drift_floor`.
 
 ### Cost
diff --git a/llms-full.txt b/llms-full.txt
index 708f239..c3b7dee 100644
--- a/llms-full.txt
+++ b/llms-full.txt
@@ -228,6 +228,8 @@ evalshift report RUN_ID [-c CONFIG] [--open] [--insights/--no-insights (default
   as their labels (max_tool_divergence -> "Tool-selection divergence"; map in
   analysis/policy.py BUDGET_LABELS) with the yaml field kept as a secondary code hint in the
   budget-failures table only. report.json and the bundle keep the machine labels.
+  decision.failure_categories counts each label per distinct (prompt, example), never per
+  row (tool_selection stamps TOOL_SELECTION_DRIFT on its conformance and divergence rows).
 evalshift compare [run flags] [--gate SEVS] [--policy-gate] [--open] [--push]
     [--insights/--no-insights (default on)]
   doctor -> run -> evaluate -> analyze -> report under one live progress display, for one
diff --git a/src/evalshift_cli/analysis/policy.py b/src/evalshift_cli/analysis/policy.py
index 7370967..b0526fa 100644
--- a/src/evalshift_cli/analysis/policy.py
+++ b/src/evalshift_cli/analysis/policy.py
@@ -1534,7 +1534,12 @@ def _blocking_regressions(comparisons: list[ComparisonResult]) -> list[BlockingR
 
 
 def _failure_categories(records: list[EvalRecord]) -> list[FailureCategoryCount]:
-    """Count each failure category over the rows that carry it.
+    """Count each failure category over the distinct examples that carry it.
+
+    Per example, not per row: one example can write several rows carrying the
+    same category -- ``tool_selection`` flags ``TOOL_SELECTION_DRIFT`` on its
+    conformance *and* its divergence row -- and the narrative built on these
+    counts reads them as "(N examples)".
 
     ``INVARIANT_VIOLATION`` is the exception: every ``trace_invariants`` row
     carries it when the target broke a rule, advisory or not, source too or
@@ -1544,13 +1549,14 @@ def _failure_categories(records: list[EvalRecord]) -> list[FailureCategoryCount]
     :func:`_invariant_violation_counts` over the blocking rows, so the two can
     never disagree.
     """
-    counter: Counter[str] = Counter()
+    examples: dict[str, set[tuple[str, str]]] = {}
     for r in records:
         raw = r.metadata.get("failure_categories", [])
         if isinstance(raw, list):
             for item in raw:
                 if isinstance(item, str) and item and item != INVARIANT_VIOLATION:
-                    counter[item] += 1
+                    examples.setdefault(item, set()).add((r.prompt_id, r.example_id))
+    counter: Counter[str] = Counter({category: len(ids) for category, ids in examples.items()})
     violated, _ = _invariant_violation_counts([r for r in records if r.blocking])
     if violated:
         counter[INVARIANT_VIOLATION] = violated
diff --git a/tests/unit/test_policy.py b/tests/unit/test_policy.py
index 6bea8d6..8935e1b 100644
--- a/tests/unit/test_policy.py
+++ b/tests/unit/test_policy.py
@@ -3016,6 +3016,27 @@ def test_advisory_breaks_alone_write_no_category(self) -> None:
         assert INVARIANT_VIOLATION not in {c.category for c in decision.failure_categories}
 
 
+class TestFailureCategoryCountsExamples:
+    def test_one_example_flagged_by_two_axes_counts_once(self) -> None:
+        """The narrative reads "Different tools chosen (N examples)".
+
+        ``tool_selection`` writes a conformance *and* a divergence row per
+        example, and both carry ``TOOL_SELECTION_DRIFT`` when the target picked
+        another tool -- counting rows read "2 examples" for one example.
+        """
+        decision = _decide(
+            [
+                _conformance_record("e1", 1.0, 0.0),
+                _divergence_record("e1", 0.0),
+                _divergence_record("e2", 0.0),
+                _conformance_record("e3", 1.0, 1.0),
+                _divergence_record("e3", 1.0),
+            ],
+        )
+        counts = {c.category: c.count for c in decision.failure_categories}
+        assert counts[TOOL_SELECTION_DRIFT] == 2
+
+
 _SKIPPED_SEMANTIC = SkippedEvaluator(
     evaluator_name="semantic.cosine",
     kind="semantic",

From 2142d0f34de19ff552ccedc2d13965e0dd9bcc4c Mon Sep 17 00:00:00 2001
From: Lukas Babaliauskas <l.babaliauskas@gmail.com>
Date: Tue, 13 Oct 2026 22:56:58 +0200
Subject: [PATCH 2/2] fix(insights): key the narrative cache on the facts it
 describes

insights.json was keyed on state.config_hash, fixed when the run is
created, while evaluate and analyze re-read evalshift.yaml every time.
Loosening a budget and re-running analyze + report rendered a PASS
verdict beside a cached "FAIL under the configured policy" paragraph.
Fold a digest of the FACTS block into the key so a change to scores,
analysis or the persisted decision regenerates it; a plain re-render
still hits.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 CHANGELOG.md                        |  7 +++++
 DOCS.md                             |  4 +--
 docs/configuration.md               |  7 +++--
 llms-full.txt                       |  9 ++++--
 src/evalshift_cli/insights/stage.py | 45 ++++++++++++++++++++++-------
 tests/unit/test_insights_stage.py   | 30 +++++++++++++++++++
 6 files changed, 85 insertions(+), 17 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index c02349c..be5474b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -14,6 +14,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   that picked another tool read "Different tools chosen (2 examples)" in the narrative
   and 2 in the report's top regression causes. Each label is now counted once per
   distinct example, as `INVARIANT_VIOLATION` already was.
+- The cached run narrative no longer outlives the figures it describes. `insights.json`
+  was keyed on the run's `config_hash`, which is fixed when the run is created, while
+  `evaluate` and `analyze` re-read `evalshift.yaml` every time: loosening a budget and
+  re-running `analyze` + `report` rendered a PASS verdict beside a cached "FAIL under the
+  configured policy" paragraph. The key now also covers the narrative's facts (scores,
+  analysis and the persisted decision), so any change to them regenerates it on the next
+  `report`; a plain re-render still reuses it. Existing caches miss once and regenerate.
 
 ## [1.4.2] - 2026-10-12
 
diff --git a/DOCS.md b/DOCS.md
index 73f61fe..7035ba4 100644
--- a/DOCS.md
+++ b/DOCS.md
@@ -785,14 +785,14 @@ CI wiring (on `analyze` and `compare`):
 
 **A FAIL's cause is read off the facts, not inferred.** Every gating budget, overall and slice rows alike, enters the prompt with its status beside its limit — `budget "<label>": limit <x>, <status>`, where the status is `breached`, `within`, `measured nothing`, or `over its limit but not confirmed at this sample size`. Beside them go the `failed_budgets` list (the breached rows, overall and slice, named as the decision prose names them: "'security' slice overall regression rate") and, when the decision carries one, its `reason` as `verdict_reason`, in plain words: the `evalshift.yaml` keys it quotes are reworded (`migration_policy.max_invariant_violations` becomes "the trace-rule violations budget", or "the 'security' slice trace-rule violations budget" where the reason quotes it beside that slice's limit, and `fail_on_dropped_params` / `drop_params` are put into words), and its figures join the permitted numbers. `breached` is the only status that may be blamed: the instruction allows only a breached budget to be named as why the run failed or as something to fix, forbids describing a `within` budget as breached, exceeded, failed, missed, violated or something to address, and requires the verdict summary to agree with `verdict_reason`. Both new keys are FACTS keys like any other, so echoing either in prose rejects the generation. A narrative once explained a FAIL as "failed to meet the budget for Trace-rule violations" on a run where that budget held and only the latency ceiling was breached: the prompt had listed limits without results, so the model guessed.
 
-On a `fail` the verdict summary and recommendation are also scanned for a held budget that shares a clause with a blame word (breach…, exceed…, fail…, violat…, address…, miss/missed/misses/missing, unmet, not met, broke, broken, and over/above/beyond followed by a limit phrase — an optional "the", "its" or "their", at most two words or figures, then a limit noun: limit, budget, ceiling, threshold, maximum, allowance or policy, singular or plural, as in "over budget" or "over the +200% cost limit" — so "+102% over the source" is not blame) that is not negated by a "not", "no", "never" or "without" in the 24 characters before it. Each mention of a held budget in that clause is judged on its own, and a mention is excused — it names the budget without accusing it — in two cases. A negation directly before it, with only other budget labels, "or", "and", "nor" or commas between, clears it when the name stands alone — the label, and any labels chained to it, is followed by a comma, the end of the clause, or "or", "and", "nor" or "but": "failed only because of Latency increase, not Cost increase or Trace-rule violations" passes, while "failed with no Cost increase headroom left" is still rejected, because there the negation belongs to the noun the label modifies. A holding phrase after it describes it when the phrase is measured against the limit — "within", "inside", "under" or "below" followed by the same limit phrase the blame words use ("within policy", "within budget", "under their limits", "under the +200% cost limit"), or a bare "held", "holds" or "passed" with nothing but punctuation after it to the end of the clause ("Trace-rule violations held.") — provided that phrase is not itself negated, no blame word stands between the two, and no other budget label does unless joined by "and", "or", "nor" or a comma. A verb or a preposition alone is not a holding phrase: "the migration failed on Latency increase alone, as Cost increase and Trace-rule violations all stayed within budget" passes, while "Cost increase remained above the +200% limit", "Cost increase held at +250%, exceeding its limit", "Cost increase under the target model breached its ceiling", "Cost increase, within hours, broke its ceiling", "Cost increase exceeded its limit and stayed within budget" and "failed on Cost increase as Latency increase stayed within its ceiling" are still rejected. Four misses are accepted: a holding phrase excuses the mention before it whatever follows, so "Cost increase stayed within budget at first, then exceeded it", "Address Cost increase so it stays within budget", "Cost increase below the target's ceiling failed the run" and "Cost increase, below the ceiling on 3 prompts, failed on the rest" pass. A held budget is blamed when one of its mentions is not excused and the clause has an un-negated blame word. Clauses split at sentence punctuation (`. ; : ! ?`), at while, but, whereas, although, though and however, at ", and", ", but", ", with" and ", however,", and at an em or en dash — not at a bare "and" or a bare comma, because "exceeded latency and failed to meet Trace-rule violations" accuses both. Every budget label, held or not, is located longest-first and masked before the scan, matching case-insensitively with hyphens, whitespace and quote characters folded: "checkout slice cost increase" is still the slice's label rather than a mention of the held "Cost increase" inside it, and "Trace-rule violations" does not read as a violation of whichever budget shares its clause. A match rejects the generation like an invented number, the retry names the misblamed budgets ("Your previous answer blamed budgets that held: …", followed by "No budget was breached" on a fail no budget caused), and two rejections ship the templated prose, which names the breached budget, or the overall blocking regressions when no budget was breached, and otherwise points the reader at the decision's recommendations and the budget table. The scan is a safety net, not a parser: it catches the shipped phrasing and its close variants, and the prompt change is the primary fix. A false match is not free: it costs the retry, which is shared with invented-number rejections, and a repeated one ships the templated prose, which has no findings. A narrative is cached per run (see below), and the cache key does not change with the CLI, so an existing run keeps its old prose until its `insights.json` is deleted.
+On a `fail` the verdict summary and recommendation are also scanned for a held budget that shares a clause with a blame word (breach…, exceed…, fail…, violat…, address…, miss/missed/misses/missing, unmet, not met, broke, broken, and over/above/beyond followed by a limit phrase — an optional "the", "its" or "their", at most two words or figures, then a limit noun: limit, budget, ceiling, threshold, maximum, allowance or policy, singular or plural, as in "over budget" or "over the +200% cost limit" — so "+102% over the source" is not blame) that is not negated by a "not", "no", "never" or "without" in the 24 characters before it. Each mention of a held budget in that clause is judged on its own, and a mention is excused — it names the budget without accusing it — in two cases. A negation directly before it, with only other budget labels, "or", "and", "nor" or commas between, clears it when the name stands alone — the label, and any labels chained to it, is followed by a comma, the end of the clause, or "or", "and", "nor" or "but": "failed only because of Latency increase, not Cost increase or Trace-rule violations" passes, while "failed with no Cost increase headroom left" is still rejected, because there the negation belongs to the noun the label modifies. A holding phrase after it describes it when the phrase is measured against the limit — "within", "inside", "under" or "below" followed by the same limit phrase the blame words use ("within policy", "within budget", "under their limits", "under the +200% cost limit"), or a bare "held", "holds" or "passed" with nothing but punctuation after it to the end of the clause ("Trace-rule violations held.") — provided that phrase is not itself negated, no blame word stands between the two, and no other budget label does unless joined by "and", "or", "nor" or a comma. A verb or a preposition alone is not a holding phrase: "the migration failed on Latency increase alone, as Cost increase and Trace-rule violations all stayed within budget" passes, while "Cost increase remained above the +200% limit", "Cost increase held at +250%, exceeding its limit", "Cost increase under the target model breached its ceiling", "Cost increase, within hours, broke its ceiling", "Cost increase exceeded its limit and stayed within budget" and "failed on Cost increase as Latency increase stayed within its ceiling" are still rejected. Four misses are accepted: a holding phrase excuses the mention before it whatever follows, so "Cost increase stayed within budget at first, then exceeded it", "Address Cost increase so it stays within budget", "Cost increase below the target's ceiling failed the run" and "Cost increase, below the ceiling on 3 prompts, failed on the rest" pass. A held budget is blamed when one of its mentions is not excused and the clause has an un-negated blame word. Clauses split at sentence punctuation (`. ; : ! ?`), at while, but, whereas, although, though and however, at ", and", ", but", ", with" and ", however,", and at an em or en dash — not at a bare "and" or a bare comma, because "exceeded latency and failed to meet Trace-rule violations" accuses both. Every budget label, held or not, is located longest-first and masked before the scan, matching case-insensitively with hyphens, whitespace and quote characters folded: "checkout slice cost increase" is still the slice's label rather than a mention of the held "Cost increase" inside it, and "Trace-rule violations" does not read as a violation of whichever budget shares its clause. A match rejects the generation like an invented number, the retry names the misblamed budgets ("Your previous answer blamed budgets that held: …", followed by "No budget was breached" on a fail no budget caused), and two rejections ship the templated prose, which names the breached budget, or the overall blocking regressions when no budget was breached, and otherwise points the reader at the decision's recommendations and the budget table. The scan is a safety net, not a parser: it catches the shipped phrasing and its close variants, and the prompt change is the primary fix. A false match is not free: it costs the retry, which is shared with invented-number rejections, and a repeated one ships the templated prose, which has no findings. A narrative is cached per run (see below), and the cache key does not include the CLI version: a newer CLI regenerates an existing run's prose only when it changes a figure the narrative describes, so a prompt or scan change alone keeps the old prose until its `insights.json` is deleted.
 
 ```yaml
 defaults:
   insights_model: gemini-3.1-flash-lite-preview   # optional; falls back to judge_model
 ```
 
-- **Cost**: one model call per run (a second only when the first generation is rejected). The narrative is cached in `insights.json` and keyed on the run's `config_hash` plus the model id, so re-running `report` or `push` costs nothing; changing either invalidates the cache and regenerates.
+- **Cost**: one model call per run (a second only when the first generation is rejected). The narrative is cached in `insights.json` and keyed on the run's `config_hash`, the model id and the figures it describes (the FACTS block, read off the run's scores, analysis and persisted decision), so re-running `report` or `push` costs nothing. Changing any of them invalidates the cache and regenerates — including a re-run of `evaluate` or `analyze` on the same run after an `evalshift.yaml` edit, which `config_hash` alone cannot see because it is fixed when the run is created. `push` ships the narrative the last `report` wrote, so run `report` after a re-`analyze` before pushing.
 - **Skipped** silently when `--no-insights` is passed, and with a warning when no API key is set for the chosen model or the run directory has no usable `evalshift.yaml`. A skip is never an error.
 - **Never fatal.** Any failure inside generation is logged and leaves the narrative empty; a run that already has its statistics is not worth failing over a missing paragraph.
 - **What gets sent to the model**: the pre-rendered figures, plus the **worst 8 regressions'** inputs and both models' outputs (each truncated to 2000 characters). That is the same exposure `llm_judge` already has, but it is real — if your suite carries data you would not send to an LLM judge, run with `--no-insights`.
diff --git a/docs/configuration.md b/docs/configuration.md
index a70310d..522b581 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -428,8 +428,11 @@ regenerated if it contains a numeric token that was not supplied. Two
 bad generations fall back to deterministic templated prose.
 
 - One model call per run (a second only on a rejected generation),
-  cached in `insights.json` and keyed on the run's `config_hash` plus
-  the model id — re-running `report` or `push` costs nothing.
+  cached in `insights.json` and keyed on the run's `config_hash`, the
+  model id and the figures the narrative describes — re-running `report`
+  or `push` costs nothing, while re-running `evaluate` or `analyze` on the
+  same run after an `evalshift.yaml` edit regenerates it on the next
+  `report`.
 - Skip it with `--insights/--no-insights` on `report` and `compare`. It is
   also skipped when no API key is configured for the chosen model, and
   when the run has no usable `evalshift.yaml`.
diff --git a/llms-full.txt b/llms-full.txt
index c3b7dee..7743c98 100644
--- a/llms-full.txt
+++ b/llms-full.txt
@@ -1252,12 +1252,15 @@ Model: defaults.insights_model, falling back to defaults.judge_model.
   Heuristic safety net by design: it catches the shipped phrasing and close variants, the
   prompt rule is the primary fix; a false match costs the retry (shared with invented-number
   rejections) and a repeated one ships the templated prose, which has no findings. Cached
-  narratives do not regenerate (key = config_hash + model, not the CLI version): delete
-  insights.json to rewrite an existing run's prose.
+  narratives regenerate only when a figure moves (key = config_hash + model + FACTS digest,
+  not the CLI version): delete insights.json to rewrite an existing run's prose.
 - Prompt input: the pre-rendered FACTS block plus the worst 8 regressions by worst_delta_score
   (input + both outputs, each truncated to 2000 chars). Same data exposure as llm_judge.
 - Cost: 1 model call per run (2 on a rejected generation). Cached in insights.json keyed on
-  state.config_hash + model id, so re-running report/push is free; either moving = cache miss.
+  state.config_hash + model id + a digest of the FACTS block, so re-running report/push is
+  free; any moving = cache miss. state.config_hash is fixed at run creation, so the FACTS
+  digest is what catches evaluate/analyze re-run on the same run after a yaml edit. push
+  ships whatever the last report cached.
 - Skipped, never an error: silently on --no-insights; with a warning when there is no provider
   API key for the chosen model or no loadable evalshift.yaml. Any generation failure is
   swallowed and leaves the narrative absent — a narrative NEVER fails a run.
diff --git a/src/evalshift_cli/insights/stage.py b/src/evalshift_cli/insights/stage.py
index aa8cdb2..fa69340 100644
--- a/src/evalshift_cli/insights/stage.py
+++ b/src/evalshift_cli/insights/stage.py
@@ -26,6 +26,7 @@
 import logging
 import os
 from collections.abc import Mapping, Sequence
+from dataclasses import asdict
 from pathlib import Path
 from typing import Any
 
@@ -94,7 +95,11 @@ def ensure_insight(
     try:
         state = read_state(run_dir)
         model = cfg.defaults.insights_model or cfg.defaults.judge_model
-        config_hash = insights_config_hash(run_config_hash=state.config_hash, model=model)
+        # Built before the lookup because the facts are part of the key.
+        facts = build_run_facts(run_dir, state=state)
+        config_hash = insights_config_hash(
+            run_config_hash=state.config_hash, model=model, facts=facts
+        )
 
         cached = load_cached_insight(run_dir, config_hash=config_hash)
         if cached is not None:
@@ -109,7 +114,6 @@ def ensure_insight(
             )
             return None
 
-        facts = build_run_facts(run_dir, state=state)
         insight = asyncio.run(
             generate_insight(facts, model=model, client=client or ModelClient()),
         )
@@ -121,16 +125,36 @@ def ensure_insight(
     return insight
 
 
-def insights_config_hash(*, run_config_hash: str, model: str) -> str:
+def insights_config_hash(*, run_config_hash: str, model: str, facts: Facts) -> str:
     """The cache key for a run's narrative.
 
     ``run_config_hash`` is the run's own ``state.config_hash`` — the canonical
-    config plus the suite path — which moves whenever anything the figures are
-    derived from moves. The model id is folded in on top because provenance is
-    part of the narrative: prose attributed to a model that no longer writes it
-    is stale even when every figure in it still holds.
+    config plus the suite path. It is fixed when the run is created, so it
+    cannot see what moves the figures afterwards: ``evaluate`` and ``analyze``
+    re-read ``evalshift.yaml`` on every invocation, and a newer CLI re-analyses
+    an old run differently. ``facts`` is everything the narrative may refer to,
+    so folding it in makes any of those a miss — a policy edit followed by
+    ``analyze`` + ``report`` once rendered "FAIL under the configured policy"
+    beside a PASS verdict block — while a plain re-render still hits. The model
+    id is folded in on top because provenance is part of the narrative: prose
+    attributed to a model that no longer writes it is stale even when every
+    figure in it still holds.
     """
-    return _sha256({"config_hash": run_config_hash, "model": model})
+    return _sha256(
+        {"config_hash": run_config_hash, "model": model, "facts": _facts_digest(facts)},
+    )
+
+
+def _facts_digest(facts: Facts) -> str:
+    """A stable hash of ``facts``; frozensets are sorted so the order is fixed."""
+    canonical = json.dumps(
+        asdict(facts),
+        sort_keys=True,
+        separators=(",", ":"),
+        ensure_ascii=False,
+        default=lambda value: sorted(value) if isinstance(value, frozenset) else str(value),
+    )
+    return "sha256:" + hashlib.sha256(canonical.encode("utf-8")).hexdigest()
 
 
 def _sha256(value: Any) -> str:
@@ -169,8 +193,9 @@ def read_bundle_insight(run_dir: Path) -> dict[str, Any] | None:
     Re-serialised through :class:`~evalshift_cli.insights.models.Insight` rather
     than handed over as read: a hand-edited cache file cannot then smuggle a
     key past the server's ``extra="forbid"`` model. Validity of the cache is
-    not checked — a narrative written for an earlier config still describes
-    this run's artifacts, and ``report`` is what refreshes it.
+    not checked here: ``report`` is what refreshes it (see
+    :func:`insights_config_hash`), so a push straight after a re-``analyze``,
+    with no ``report`` between, ships the narrative the last report wrote.
     """
     envelope = _read_envelope(run_dir)
     if envelope is None:
diff --git a/tests/unit/test_insights_stage.py b/tests/unit/test_insights_stage.py
index c4d3aad..46a3db9 100644
--- a/tests/unit/test_insights_stage.py
+++ b/tests/unit/test_insights_stage.py
@@ -188,6 +188,36 @@ def test_a_changed_insights_model_invalidates_the_cache(
     assert fake_client.call_count == 2
 
 
+def test_a_reanalysed_decision_invalidates_the_cache(
+    project: RunFixture, fake_client: FakeModelClient
+) -> None:
+    """A policy edit then ``analyze`` + ``report`` on the same run.
+
+    ``state.config_hash`` is fixed when the run is created, so it cannot see the
+    edit: the report rendered the new verdict beside the cached narrative's old
+    one -- "FAIL under the configured policy" next to a PASS verdict block.
+    """
+    _report(project, client=fake_client)
+    _write_policy(project, "  max_overall_regression_rate: 0.0\n")
+    _persist_decision(project)
+
+    _report(project, client=fake_client)
+    assert fake_client.call_count == 2
+
+
+def test_rescored_records_invalidate_the_cache(
+    project: RunFixture, fake_client: FakeModelClient
+) -> None:
+    """``evaluate`` again on the same run (a new key, an edited evaluator)."""
+    _report(project, client=fake_client)
+    scores = project.run_dir / "scores.jsonl"
+    rows = [line for line in scores.read_text(encoding="utf-8").splitlines() if line.strip()]
+    scores.write_text("\n".join(rows[:-1]) + "\n", encoding="utf-8")
+
+    _report(project, client=fake_client)
+    assert fake_client.call_count == 2
+
+
 # ---------------------------------------------------------------------------
 # The envelope
 # ---------------------------------------------------------------------------