feat(grafana-alertcheck): rename outcomes and print plain-worded tables

grafana-alertcheck/.changeset/v0.1.3.md

@@ -0,0 +1,2 @@
+- The `check` output now speaks plain words. Outcome values are renamed: `clean` → `healthy`, `newly_bad` → `new_failure`, `persistently_bad` → `still_failing`, `flapping` → `unstable`, `skipped` → `paused`, `unobservable` → `not_verified`, and the early-exit `terminated_early.kind` follows the same rename. A `--min-observed` deficit that no rule explains is now reported as `not_counted` instead of being blamed on a paused rule. This changes the `--output json` vocabulary and anything downstream of it, including the action's `outcomes` output.
+- The human table dropped its internal column names. `RESULTS` is now `ALERT`/`VERDICT`/`BROKEN FOR`/`CHECKED EVERY`/`WINDOW COVERED`/`DETAILS`; `VIOLATIONS` uses `GRAFANA STATE`/`GRAFANA HEALTH` and a single-word `INSTANCES` column (the previous `INSTANCE COUNT` header read as two columns, one of them empty); `THRESHOLDS` became `LIMITS USED`, with limits named in plain words and explained by a legend under the table. The footer now spells out the extra observation time, the evaluation wait and the clock difference from Grafana.

grafana-alertcheck/cmd/table.go

@@ -11,22 +11,29 @@ import (
 	"github.com/smartcontractkit/chainlink-testing-framework/grafana-alertcheck/internal/gate"
 )
 
+// limitsLegend explains each LIMITS USED column in one plain sentence, so the
+// table needs no documentation lookup.
+const limitsLegend = `  max gap without check — the longest gap between two checks we accept before we say the alert was not watched.
+  query failing for — how long Grafana may keep failing to run the alert's query before we stop trusting its state.
+  no evaluation for — how long Grafana may go without evaluating the alert before we stop trusting its state.`
+
 // renderTable is the human table. It always writes to the writer it is given,
 // which the caller (runCheck) always points at stderr — stdout is reserved for
 // the machine-readable --output json.
 //
-// Three titled tables, in order (the name column is RULE in all of them — one
-// row is one resolved alert rule, never a firing instance):
+// Three titled tables, in order (the ALERT column is one resolved alert rule,
+// never a firing instance):
+//
+//  1. RESULTS, one line per rule: verdict, time broken, check cadence,
+//     whether the window was observed, and any notes;
+//  2. VIOLATIONS, one line per distinct violation, with the raw Grafana state
+//     and health and the number of instances it stands for;
+//  3. LIMITS USED, the coverage thresholds that answer "why" on exit 2. Each
+//     column is named in plain words and explained by the legend below it, so
+//     the table needs no documentation lookup.
 //
-//  1. RESULTS, one line per rule: outcome, BadFor, pollEvery, proved-or-not
-//     with the largest gap;
-//  2. VIOLATIONS, one line per distinct violation.
-//  3. THRESHOLDS, the numbers that answer "why" on exit 2: each non-skipped
-//     rule's maxGap/healthGrace/evalStaleAfter, followed by the global
-//     transitionGrace and drainTimeout, and the largest measured clock skew
-//     alongside its own error bound (RTT/2) — SkewHardLimit is a separate,
-//     fixed input threshold and is reported next to it, never as if it were
-//     that bound.
+// The global footer then reports the extra observation time, the drain limit
+// and the largest measured clock difference, also in plain words.
 func renderTable(w io.Writer, res gate.Result) error {
 	alertOf := make(map[string]string, len(res.Verdicts))
 	for _, v := range res.Verdicts {
@@ -52,7 +59,7 @@ func renderTable(w io.Writer, res gate.Result) error {
 
 	fmt.Fprintln(w, "RESULTS")
 	tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
-	fmt.Fprintln(tw, "RULE\tOUTCOME\tBADFOR\tPOLLEVERY\tPROVED\tNOTE")
+	fmt.Fprintln(tw, "ALERT\tVERDICT\tBROKEN FOR\tCHECKED EVERY\tWINDOW COVERED\tDETAILS")
 	for _, v := range sortedVerdicts(res.Verdicts) {
 		fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n",
 			v.Alert, v.Outcome, v.BadFor.Round(time.Second), v.PollEvery.Round(time.Second),
@@ -65,7 +72,7 @@ func renderTable(w io.Writer, res gate.Result) error {
 	if len(res.Violations) > 0 {
 		fmt.Fprintln(w, "\nVIOLATIONS")
 		vtw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
-		fmt.Fprintln(vtw, "RULE\tOUTCOME\tSTATE\tHEALTH\tINSTANCE COUNT\tNOTE")
+		fmt.Fprintln(vtw, "ALERT\tVERDICT\tGRAFANA STATE\tGRAFANA HEALTH\tINSTANCES\tDETAILS")
 		for _, g := range groupedViolations(res.Violations) {
 			fmt.Fprintf(vtw, "%s\t%s\t%s\t%s\t%s\t%s\n", alertLabel(g.v, alertOf), g.v.Outcome, g.v.State, g.v.Health, instanceCount(g), g.v.Note)
 		}
@@ -74,14 +81,13 @@ func renderTable(w io.Writer, res gate.Result) error {
 		}
 	}
 
-	// The per-rule thresholds answer "why" on exit 2: a table, not the prose
-	// "rule NAME: maxGap=... healthGrace=... evalStaleAfter=..." that repeated
-	// the rule name a fourth time. It is separated from the result above by a
-	// blank line.
+	// The per-rule limits answer "why" on exit 2. The columns are spelled out
+	// and explained by limitsLegend right below, so an operator does not have
+	// to look anything up.
 	fmt.Fprintln(w)
-	fmt.Fprintln(w, "THRESHOLDS")
+	fmt.Fprintln(w, "LIMITS USED")
 	ttw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0)
-	fmt.Fprintln(ttw, "RULE\tMAXGAP\tHEALTHGRACE\tEVALSTALEAFTER")
+	fmt.Fprintln(ttw, "ALERT\tMAX GAP WITHOUT CHECK\tQUERY FAILING FOR\tNO EVALUATION FOR")
 	for _, uid := range sortedThresholdUIDs(res.Thresholds, alertOf) {
 		t := res.Thresholds[uid]
 		fmt.Fprintf(ttw, "%s\t%s\t%s\t%s\n",
@@ -90,11 +96,17 @@ func renderTable(w io.Writer, res gate.Result) error {
 	if err := ttw.Flush(); err != nil {
 		return fmt.Errorf("render table: %w", err)
 	}
+	fmt.Fprintln(w, limitsLegend)
 
 	fmt.Fprintln(w)
-	fmt.Fprintf(w, "global: transitionGrace=%s (source: %s) drainTimeout=%s\n",
-		res.Global.TransitionGrace, res.Global.GraceSource, res.Global.DrainTimeout)
-	fmt.Fprintf(w, "largest measured clock skew: %s (bound ±%s, hard limit %s), grafana %s\n",
+	if res.Global.TransitionGrace > 0 {
+		fmt.Fprintf(w, "extra watching after your window: +%s — so an alert that only starts firing at the end is still caught (slowest: %s)\n",
+			res.Global.TransitionGrace, res.Global.GraceSource)
+	} else {
+		fmt.Fprintln(w, "extra watching after your window: none")
+	}
+	fmt.Fprintf(w, "max wait for all alerts to finish evaluating: %s\n", res.Global.DrainTimeout)
+	fmt.Fprintf(w, "clock difference from Grafana: %s, accurate to ±%s (checks fail above %s); Grafana %s\n",
 		res.ClockSkew.Round(time.Millisecond), res.ClockSkewBound.Round(time.Millisecond),
 		gate.SkewHardLimit, res.GrafanaVersion)
 	// The verdict — the single number a terminal operator reads last — sits on
@@ -104,8 +116,8 @@ func renderTable(w io.Writer, res gate.Result) error {
 	return nil
 }
 
-// violationsLabel colours the "violations: N" prefix of the footer: green for a
-// clean run, red otherwise. The rest of the line is written uncoloured.
+// violationsLabel colours the "violations: N" prefix of the footer: green when
+// there are none, red otherwise. The rest of the line is written uncoloured.
 func violationsLabel(n int, enabled bool) string {
 	s := fmt.Sprintf("violations: %d", n)
 	if !enabled {
@@ -117,10 +129,10 @@ func violationsLabel(n int, enabled bool) string {
 	return ansiRed + s + ansiReset
 }
 
-// provedLabel is the table's PROVED column: "yes" for a clean coverage
-// proof, "no" with the reason and largest gap for an unobservable rule, and
-// "-" for a rule decide never asked proveCoverage about at all (skipped —
-// paused before the window opened).
+// provedLabel is the table's WINDOW COVERED column: "yes" for a fully
+// observed window, "no" with the reason and largest gap for a not-verified
+// rule, and "-" for a rule decide never asked proveCoverage about at all
+// (paused before the window opened).
 func provedLabel(cov gate.CoverageResult) string {
 	if cov.Reason == "" && !cov.Unobservable && !cov.Proved {
 		return "-"
@@ -192,8 +204,11 @@ func sameRendered(a, b gate.Violation) bool {
 	return violationSignature(a) == violationSignature(b)
 }
 
+// instanceCount is the INSTANCES column: how many alert instances one grouped
+// violation row stands for. Paused and not-counted rows stand for no instance
+// at all, so they render "-".
 func instanceCount(g violationGroup) string {
-	if g.v.Outcome == gate.OutcomeSkipped {
+	if g.v.Outcome == gate.OutcomePaused || g.v.Outcome == gate.OutcomeNotCounted {
 		return "-"
 	}
 	return strconv.Itoa(g.n)

grafana-alertcheck/cmd/table_test.go

@@ -11,24 +11,24 @@ import (
 )
 
 // The golden table test: a fixed Result renders a deterministic, ordered rule
-// table, a violations section and a footer carrying the per-rule and global
-// thresholds plus the skew and its bound — with no live Check involved.
+// table, a violations section and a footer carrying the per-rule limits and
+// globals in plain words — with no live Check involved.
 func TestRenderTable(t *testing.T) {
 	gapAt := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
 	res := gate.Result{
 		GrafanaVersion: "13.1.0",
 		ClockSkew:      1500 * time.Millisecond,
 		ClockSkewBound: 250 * time.Millisecond,
 		Verdicts: []gate.RuleVerdict{
-			{Alert: "Zebra Alert", RuleUID: "uid-z", Outcome: gate.OutcomeClean, PollEvery: 30 * time.Second},
-			{Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeUnobservable,
+			{Alert: "Zebra Alert", RuleUID: "uid-z", Outcome: gate.OutcomeHealthy, PollEvery: 30 * time.Second},
+			{Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeNotVerified,
 				PollEvery: 30 * time.Second, Note: "gap of 5m0s starting at 2026-01-01T12:00:00Z exceeds maxGap 1m0s"},
-			{Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomeSkipped,
+			{Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomePaused,
 				Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set"},
 		},
 		Violations: []gate.Violation{
-			{Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeUnobservable, State: gate.StateFiring, Health: "error", Note: "unobservable"},
-			{Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomeSkipped,
+			{Alert: "Ape Alert", RuleUID: "uid-a", Outcome: gate.OutcomeNotVerified, State: gate.StateFiring, Health: "error", Note: "not verified"},
+			{Alert: "Paused Alert", RuleUID: "uid-p", Outcome: gate.OutcomePaused,
 				Note: "paused before the window opened; counts against --min-observed unless --allow-paused is set"},
 		},
 		Coverage: map[string]gate.CoverageResult{
@@ -50,55 +50,62 @@ func TestRenderTable(t *testing.T) {
 	require.NoError(t, renderTable(&buf, res))
 	out := buf.String()
 
-	// Rule table: Ape sorts before Zebra sorts before... Paused is skipped and
-	// carries no coverage entry, so it renders "-" for PROVED.
+	// The rule table: Ape sorts before Zebra sorts before... Paused carries no
+	// coverage entry, so it renders "-" for WINDOW COVERED.
+	require.Contains(t, out, "ALERT")
 	require.Contains(t, out, "Ape Alert")
-	require.Contains(t, out, "unobservable")
+	require.Contains(t, out, "not_verified")
 	require.Contains(t, out, "heartbeat_gap")
 	require.Contains(t, out, "largest gap 5m0s")
 	require.Contains(t, out, "Zebra Alert")
-	require.Contains(t, out, "clean")
+	require.Contains(t, out, "healthy")
+	require.Contains(t, out, "WINDOW COVERED")
 
 	// The violations section must show up even without --output json, and must
-	// carry the --allow-paused hint text verbatim.
+	// carry the --allow-paused hint text verbatim. INSTANCES is a single word
+	// so the count under it cannot read as a second, empty column.
 	require.Contains(t, out, "VIOLATIONS")
 	require.Contains(t, out, "--allow-paused")
-	require.Contains(t, out, "STATE")
-	require.Contains(t, out, "HEALTH")
-	require.Contains(t, out, "INSTANCE COUNT")
+	require.Contains(t, out, "GRAFANA STATE")
+	require.Contains(t, out, "GRAFANA HEALTH")
+	require.Contains(t, out, "INSTANCES")
 	require.Contains(t, out, string(gate.StateFiring))
 	require.Contains(t, out, "error")
 
-	// The footer: per-rule thresholds are a table (RULE/MAXGAP/HEALTHGRACE/
-	// EVALSTALEAFTER) rather than prose, followed by the global thresholds and
-	// the violations count with the skew and its own bound rather than the
-	// fixed hard limit.
-	require.Contains(t, out, "MAXGAP")
-	require.Contains(t, out, "HEALTHGRACE")
-	require.Contains(t, out, "EVALSTALEAFTER")
-	require.Contains(t, out, "global: transitionGrace=5m0s (source: Ape Alert (for=5m)) drainTimeout=2m0s")
-	require.Contains(t, out, "largest measured clock skew: 1.5s (bound ±250ms, hard limit 1m0s)")
+	// The limits table names each threshold in plain words and explains it
+	// right below, so an operator does not have to consult the docs.
+	require.Contains(t, out, "LIMITS USED")
+	require.Contains(t, out, "MAX GAP WITHOUT CHECK")
+	require.Contains(t, out, "QUERY FAILING FOR")
+	require.Contains(t, out, "NO EVALUATION FOR")
+	require.Contains(t, out, "the longest gap between two checks")
+	require.Contains(t, out, "without evaluating the alert")
+
+	// The global footer in plain words.
+	require.Contains(t, out, "extra watching after your window: +5m0s")
+	require.Contains(t, out, "slowest: Ape Alert (for=5m)")
+	require.Contains(t, out, "max wait for all alerts to finish evaluating: 2m0s")
+	require.Contains(t, out, "clock difference from Grafana: 1.5s, accurate to ±250ms (checks fail above 1m0s); Grafana 13.1.0")
 	require.Contains(t, out, "violations: 2")
-	require.Contains(t, out, "13.1.0")
 }
 
 // The "-" case: a rule decide never asked proveCoverage about (paused before
 // the window opened) has an empty CoverageResult and must not be reported as
-// either proved or unobservable.
-func TestProvedLabel_Skipped(t *testing.T) {
+// either covered or not verified.
+func TestProvedLabel_Paused(t *testing.T) {
 	require.Equal(t, "-", provedLabel(gate.CoverageResult{}))
 }
 
 // groupedViolations collapses a rule's many firing instances into one row per
 // rendered signature, each with a count.
 func TestGroupedViolations(t *testing.T) {
 	in := []gate.Violation{
-		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"},
-		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"},
-		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "ok"},
-		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomePersistentlyBad, State: gate.StateFiring, Health: "error"},
-		{Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewlyBad, State: gate.StateFiring, Health: "ok", Note: "x"},
-		{Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewlyBad, State: gate.StateFiring, Health: "ok", Note: "x"},
+		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomeStillFailing, State: gate.StateFiring, Health: "ok"},
+		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomeStillFailing, State: gate.StateFiring, Health: "ok"},
+		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomeStillFailing, State: gate.StateFiring, Health: "ok"},
+		{Alert: "OCR2 Consensus failure", RuleUID: "uid-o", Outcome: gate.OutcomeStillFailing, State: gate.StateFiring, Health: "error"},
+		{Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewFailure, State: gate.StateFiring, Health: "ok", Note: "x"},
+		{Alert: "Other Alert", RuleUID: "uid-p", Outcome: gate.OutcomeNewFailure, State: gate.StateFiring, Health: "ok", Note: "x"},
 	}
 
 	got := groupedViolations(in)
@@ -108,7 +115,7 @@ func TestGroupedViolations(t *testing.T) {
 	for _, g := range got {
 		counts[g.v.Health+"|"+string(g.v.Outcome)] = g.n
 	}
-	require.Equal(t, 3, counts["ok|"+string(gate.OutcomePersistentlyBad)])
-	require.Equal(t, 1, counts["error|"+string(gate.OutcomePersistentlyBad)])
-	require.Equal(t, 2, counts["ok|"+string(gate.OutcomeNewlyBad)])
+	require.Equal(t, 3, counts["ok|"+string(gate.OutcomeStillFailing)])
+	require.Equal(t, 1, counts["error|"+string(gate.OutcomeStillFailing)])
+	require.Equal(t, 2, counts["ok|"+string(gate.OutcomeNewFailure)])
 }

grafana-alertcheck/docs/architecture.md

@@ -15,11 +15,11 @@ This page documents the invariants and seams a maintainer must not break. It exi
 The gate must fail if it cannot get an answer. Every rule below is a specific instance of that:
 
 - **An error is never a pass.** A pass is exactly `len(Violations) == 0 && err == nil`. Every error path leaves `err` non-nil, and the CLI maps that to exit `2` unconditionally.
-- **Inability beats violation.** Any `unobservable` rule is exit `2`, even alongside a real violation found first.
+- **Inability beats violation.** Any `not_verified` rule is exit `2`, even alongside a real violation found first.
 - **Absent never means normal.** An instance that leaves the bad set is looked up in the *same* response: present as `normal` → cleared; absent (or `MissingSeries`) → vanished (a discontinuity, not a recovery).
 - **Staleness is absolute.** `grafana_now − lastEvaluation` is compared against a threshold, never "did it increase since the last poll" — a delta check reports stale on ~half the polls of a healthy rule (we poll at half of `intervalSeconds` of each rule).
 - **`grafana_now` is the response `Date` header.** Never the runner clock, in any comparison against a Grafana timestamp.
-- **An early exit can never be a pass.** `check` may stop collecting before `to + transitionGrace` (fail-fast), but only on a *monotone* terminal verdict: an inability that has already happened, or a post-`from` bad onset (which the full classifier would call `newly_bad`/`flapping`). The one outcome that forgives an observed bad state, `recovered`, is reserved for bad-at-`from`, so a preexisting condition is never terminal. `--no-fail-fast` removes the guard entirely.
+- **An early exit can never be a pass.** `check` may stop collecting before `to + transitionGrace` (fail-fast), but only on a *monotone* terminal verdict: an inability that has already happened, or a post-`from` bad onset (which the full classifier would call `new_failure`/`unstable`). The one outcome that forgives an observed bad state, `recovered`, is reserved for bad-at-`from`, so a preexisting condition is never terminal. `--no-fail-fast` removes the guard entirely.
 - **No replay.** No run-id key, no artifact download, no state between attempts. A retry is a new piece of work and observation.
 
 ## The pure-function seam