fix(warehouse): label an exception-less error span from its span attributes

apps/api/src/chat/prompts.ts

@@ -121,7 +121,7 @@ If you write "unknown" in \`suspectedCause\`, you MUST populate \`ruledOut\` wit
 
 The same applies when you DO name a cause: \`ruledOut\` is what makes the named cause believable. A responder reading your report should be able to see what else you considered.
 
-Never report a bare label as a cause. "Unknown Error" is a grouping label for spans with no exception and no status message — it is the *name* of the thing you were asked to explain, not an explanation of it.
+Never report a bare label as a cause. "Unknown Error" is a grouping label for spans with no exception event, no exception.*/error.* attributes and no status message — it is the *name* of the thing you were asked to explain, not an explanation of it.
 
 ## After diagnosing
 Stay in the conversation. Answer follow-up questions using the same tools, referencing the evidence you already gathered. When the user asks you to act — create an alert, transition an issue, propose a fix — call the matching mutating tool; it is approval-gated (see below).

apps/api/src/services/warehouse/error-events-attribute-fallback.clickhouse.e2e.test.ts

@@ -0,0 +1,301 @@
+// SAFETY-FILE: JSON in this test is emitted by the fixture or unit under test before its fields are asserted.
+// error_events label + fingerprint derivation for spans with no exception event.
+//
+// Cloudflare's native Workers tracing (`telemetry.sdk.name = workers-observability`)
+// records no span events, no status description and has no outcome setter — a
+// custom span can only setAttribute(). `error_events_mv` used to read the
+// exception from the first OTel `exception` span event alone, then
+// StatusMessage, then the literal 'Unknown Error', so every error span such a
+// Worker exported hashed to a single "Unknown Error" issue per service. The MV
+// now falls back to `exception.*` span attributes, then `error.type` /
+// `error.message`, before StatusMessage. This is the test that says so against
+// a real ClickHouse, through the real migration set, for both target tables.
+
+import { afterAll, assert, beforeAll, describe, it } from "@effect/vitest"
+import {
+	applyRealMigrations,
+	clickhouseE2eEnabled,
+	clickhouseExec,
+	uniqueDatabase,
+} from "./clickhouse-e2e-support"
+
+const database = uniqueDatabase("maple_error_events_attrs_e2e")
+const ORG_ID = "org_error_events_attrs"
+const SERVICE = "cf-worker"
+
+/**
+ * Now-relative, not a fixed date: `traces` and both error tables enforce a TTL
+ * at insert time, and a hardcoded timestamp silently drops every seed once it
+ * ages past the horizon, leaving the suite comparing nothing to nothing.
+ */
+const SEED_MS = Date.now() - 60 * 60 * 1000
+const chDateTime = (epochMs: number): string => new Date(epochMs).toISOString().replace("T", " ").slice(0, 19)
+const SEED_TS = chDateTime(SEED_MS)
+
+const quote = (value: string): string => `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`
+const chMap = (entries: Readonly<Record<string, string>>): string => {
+	const pairs = Object.entries(entries).flatMap(([key, value]) => [quote(key), quote(value)])
+	return pairs.length === 0 ? "map()" : `map(${pairs.join(", ")})`
+}
+
+interface ExceptionEvent {
+	readonly type: string
+	readonly message: string
+	readonly stacktrace: string
+}
+
+interface SeedSpan {
+	readonly spanId: string
+	readonly kind: "Client" | "Server"
+	readonly statusMessage: string
+	readonly spanAttributes: Readonly<Record<string, string>>
+	readonly exceptionEvent?: ExceptionEvent
+	/** Whether the tracer is Cloudflare's native one; the default is the OTel SDK. */
+	readonly native?: boolean
+}
+
+const WORKERS_OBSERVABILITY = { "telemetry.sdk.name": "workers-observability" }
+const OTEL_SDK = { "telemetry.sdk.name": "opentelemetry" }
+
+const SEED_SPANS: ReadonlyArray<SeedSpan> = [
+	// The case this file exists for: no event, no status description, only
+	// semconv error.* attributes.
+	{
+		spanId: "cf-error-type",
+		kind: "Server",
+		statusMessage: "",
+		native: true,
+		spanAttributes: {
+			"error.type": "TypeError",
+			"error.message": "Cannot read properties of undefined (reading 'id')",
+			"http.request.method": "GET",
+		},
+	},
+	// Same type, different message: must be a different issue, not one
+	// "TypeError" bucket per service.
+	{
+		spanId: "cf-error-type-other-bug",
+		kind: "Server",
+		statusMessage: "",
+		native: true,
+		spanAttributes: {
+			"error.type": "TypeError",
+			"error.message": "Cannot read properties of null (reading 'headers')",
+		},
+	},
+	// Same bug, different id in the message: the redacted signature groups them.
+	{
+		spanId: "cf-error-type-same-bug",
+		kind: "Server",
+		statusMessage: "",
+		native: true,
+		spanAttributes: {
+			"error.type": "TypeError",
+			"error.message": "Cannot read properties of undefined (reading 'id')",
+			"user.id": "u_1234567890",
+		},
+	},
+	// exception.* attributes win over error.*, and the stacktrace attribute
+	// feeds the frame portion of the hash.
+	{
+		spanId: "cf-exception-attrs",
+		kind: "Server",
+		statusMessage: "",
+		native: true,
+		spanAttributes: {
+			"exception.type": "RangeError",
+			"exception.message": "offset 4096 is out of range",
+			"exception.stacktrace":
+				"RangeError: offset 4096 is out of range\n    at slice (worker.js:1542:13655)",
+			"error.type": "LosesToException",
+			"error.message": "must not be read",
+		},
+	},
+	// A real exception event keeps exactly the precedence it always had, even
+	// when attributes disagree with it.
+	{
+		spanId: "event-wins",
+		kind: "Server",
+		statusMessage: "status text",
+		spanAttributes: {
+			"exception.type": "AttrError",
+			"error.type": "AttrError2",
+			"error.message": "attr",
+		},
+		exceptionEvent: {
+			type: "EventError",
+			message: "from the event",
+			stacktrace: "    at handler (/app/src/routes/user.ts:17:21)",
+		},
+	},
+	// The StatusMessage fallback is unchanged for spans with nothing else.
+	{
+		spanId: "status-only",
+		kind: "Server",
+		statusMessage: "DatabaseError: connection reset",
+		spanAttributes: {},
+	},
+	// StatusMessage still supplies the message text when it is set, so an
+	// event-less span that already had one keeps its hash.
+	{
+		spanId: "status-and-error-type",
+		kind: "Server",
+		statusMessage: "connection reset",
+		spanAttributes: { "error.type": "TimeoutError", "error.message": "attribute message" },
+	},
+	// Nothing carries an exception: still the Unknown Error bucket.
+	{
+		spanId: "unknown",
+		kind: "Server",
+		statusMessage: "",
+		native: true,
+		spanAttributes: { "http.request.method": "GET" },
+	},
+	// The 0016 guard: a 4xx client span whose only error.type is the status code
+	// (HTTP semconv sets that on any non-2xx response) is bot noise, not an error.
+	{
+		spanId: "bot-404",
+		kind: "Client",
+		statusMessage: "",
+		native: true,
+		spanAttributes: { "http.response.status_code": "404", "error.type": "404", "url.path": "/wp-admin" },
+	},
+	// ...but a 4xx carrying a real exception type is still an error.
+	{
+		spanId: "real-4xx",
+		kind: "Client",
+		statusMessage: "",
+		native: true,
+		spanAttributes: { "http.response.status_code": "400", "error.type": "ValidationError" },
+	},
+]
+
+const seed = async (): Promise<void> => {
+	const rows = SEED_SPANS.map((row) => {
+		const resource = chMap({
+			"service.version": "e2e",
+			"deployment.environment.name": "production",
+			...(row.native === true ? WORKERS_OBSERVABILITY : OTEL_SDK),
+		})
+		const events =
+			row.exceptionEvent === undefined
+				? "[], [], []"
+				: `[toDateTime64(${quote(SEED_TS)}, 9)], ['exception'], [${chMap({
+						"exception.type": row.exceptionEvent.type,
+						"exception.message": row.exceptionEvent.message,
+						"exception.stacktrace": row.exceptionEvent.stacktrace,
+					})}]`
+		return `(${quote(ORG_ID)}, ${quote(SEED_TS)}, ${quote(`trace-${row.spanId}`)}, ${quote(row.spanId)}, '', 'GET /', ${quote(row.kind)}, ${quote(SERVICE)}, 1000000, 'Error', ${quote(row.statusMessage)}, ${chMap(row.spanAttributes)}, ${resource}, ${events})`
+	}).join(",\n")
+
+	await clickhouseExec(
+		`INSERT INTO traces
+		 (OrgId, Timestamp, TraceId, SpanId, ParentSpanId, SpanName, SpanKind, ServiceName, Duration, StatusCode, StatusMessage, SpanAttributes, ResourceAttributes, EventsTimestamp, EventsName, EventsAttributes)
+		 VALUES\n${rows}`,
+		database,
+	)
+}
+
+interface ErrorEventRow {
+	readonly SpanId: string
+	readonly ErrorLabel: string
+	readonly ExceptionType: string
+	readonly ExceptionMessage: string
+	readonly ExceptionStacktrace: string
+	readonly TopFrame: string
+	readonly FingerprintHash: string
+}
+
+const readErrorEvents = async (
+	table: "error_events" | "error_events_by_time",
+): Promise<Map<string, ErrorEventRow>> => {
+	const body = await clickhouseExec(
+		`SELECT SpanId, ErrorLabel, ExceptionType, ExceptionMessage, ExceptionStacktrace, TopFrame, toString(FingerprintHash) AS FingerprintHash
+		 FROM ${table}
+		 WHERE OrgId = ${quote(ORG_ID)}
+		 ORDER BY SpanId
+		 FORMAT JSONEachRow`,
+		database,
+	)
+	const rows = body
+		.split("\n")
+		.filter((line) => line.trim().length > 0)
+		.map((line) => JSON.parse(line) as ErrorEventRow)
+	return new Map(rows.map((row) => [row.SpanId, row]))
+}
+
+const mustGet = (rows: Map<string, ErrorEventRow>, spanId: string): ErrorEventRow => {
+	const row = rows.get(spanId)
+	assert.isDefined(row, `expected ${spanId} to be materialized into error_events`)
+	return row
+}
+
+describe.skipIf(!clickhouseE2eEnabled)("error_events attribute fallback (ClickHouse e2e)", () => {
+	let rows: Map<string, ErrorEventRow>
+
+	beforeAll(async () => {
+		await clickhouseExec(`CREATE DATABASE IF NOT EXISTS ${database}`)
+		await applyRealMigrations(database)
+		await seed()
+		rows = await readErrorEvents("error_events")
+	}, 180_000)
+
+	afterAll(async () => {
+		await clickhouseExec(`DROP DATABASE IF EXISTS ${database}`)
+	})
+
+	it("labels a workers-observability span from its error.* attributes", () => {
+		const row = mustGet(rows, "cf-error-type")
+		assert.strictEqual(row.ErrorLabel, "TypeError")
+		assert.strictEqual(row.ExceptionType, "TypeError")
+		assert.strictEqual(row.ExceptionMessage, "Cannot read properties of undefined (reading 'id')")
+		assert.strictEqual(row.ExceptionStacktrace, "")
+		assert.notStrictEqual(row.FingerprintHash, mustGet(rows, "unknown").FingerprintHash)
+	})
+
+	it("separates two bugs of the same type in one Worker, and groups one bug across ids", () => {
+		const bug = mustGet(rows, "cf-error-type")
+		assert.notStrictEqual(bug.FingerprintHash, mustGet(rows, "cf-error-type-other-bug").FingerprintHash)
+		assert.strictEqual(bug.FingerprintHash, mustGet(rows, "cf-error-type-same-bug").FingerprintHash)
+	})
+
+	it("reads exception.* attributes ahead of error.*, stacktrace included", () => {
+		const row = mustGet(rows, "cf-exception-attrs")
+		assert.strictEqual(row.ErrorLabel, "RangeError")
+		assert.strictEqual(row.ExceptionMessage, "offset 4096 is out of range")
+		assert.include(row.ExceptionStacktrace, "at slice")
+		assert.strictEqual(row.TopFrame, "    at slice (worker.js)")
+	})
+
+	it("keeps the exception event's precedence when a span has one", () => {
+		const row = mustGet(rows, "event-wins")
+		assert.strictEqual(row.ErrorLabel, "EventError")
+		assert.strictEqual(row.ExceptionMessage, "from the event")
+		assert.strictEqual(row.TopFrame, "    at handler (/app/src/routes/user.ts)")
+	})
+
+	it("keeps the StatusMessage and Unknown Error fallbacks for everything else", () => {
+		assert.strictEqual(mustGet(rows, "status-only").ErrorLabel, "DatabaseError")
+		assert.strictEqual(mustGet(rows, "status-only").ExceptionMessage, "DatabaseError: connection reset")
+		assert.strictEqual(mustGet(rows, "unknown").ErrorLabel, "Unknown Error")
+	})
+
+	it("prefers the attribute type but the StatusMessage text when both are set", () => {
+		const row = mustGet(rows, "status-and-error-type")
+		assert.strictEqual(row.ErrorLabel, "TimeoutError")
+		assert.strictEqual(row.ExceptionMessage, "attribute message")
+	})
+
+	it("still drops a 4xx client span whose only error.type is the status code", () => {
+		assert.isUndefined(rows.get("bot-404"))
+		assert.strictEqual(mustGet(rows, "real-4xx").ErrorLabel, "ValidationError")
+	})
+
+	it("writes the same projection to error_events_by_time", async () => {
+		const byTime = await readErrorEvents("error_events_by_time")
+		assert.deepStrictEqual([...byTime.keys()], [...rows.keys()])
+		for (const [spanId, row] of rows) {
+			assert.deepStrictEqual(byTime.get(spanId), row, `error_events_by_time disagrees on ${spanId}`)
+		}
+	})
+})

apps/cli/src/server/local-schema-history.ts

@@ -181,4 +181,19 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray<LocalSchemaHistoryEntry> = Obje
 		manifestDigest: "f7d559f0db216379db02bc78c4e50f180f40588e124adf65665bf7eaaf556735",
 		projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a",
 	}),
+	Object.freeze({
+		// v17 rebuilds error_events_mv / error_events_by_time_mv so a span with
+		// no `exception` event is labelled from its exception.* / error.* span
+		// attributes (ClickHouse migration 0027). No part is rewritten and no row
+		// moves; rows already materialized keep their 'Unknown Error' label.
+		//
+		// projectRevision is carried forward deliberately — it is a hardcoded
+		// constant that no longer tracks the generator's header, and the identity
+		// this gate compares is the fingerprint/digest pair.
+		version: 17,
+		fingerprint: "9135b2a26d977584",
+		digest: "9135b2a26d977584d621f4d3ba8cce1def47d98d55a3bf53dac493f93d2e0118",
+		manifestDigest: "2ef489448e313a603e842824c5918df76ced5fb509163f101a6288194dbbc552",
+		projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a",
+	}),
 ] as const)

apps/cli/src/server/local-schema-version.ts

@@ -1,4 +1,4 @@
 // Increment this value for every structural change to the generated local
 // schema. The compatibility manifest and migration registry must be updated in
 // the same change before a new value can ship.
-export const LOCAL_SCHEMA_VERSION = 16 as const
+export const LOCAL_SCHEMA_VERSION = 17 as const

apps/cli/src/server/local-store-migrations.ts

@@ -52,6 +52,7 @@ import { v12ToV13ServiceOperationsDiscriminatorsModule } from "./local-store-mig
 import { v13ToV14AiTraceIndexModule } from "./local-store-migrations/v13-to-v14-ai-trace-index"
 import { v14ToV15CommitShaVcsRevisionModule } from "./local-store-migrations/v14-to-v15-commit-sha-vcs-revision"
 import { v15ToV16AiTraceIndexFilterColumnsModule } from "./local-store-migrations/v15-to-v16-ai-trace-index-filter-columns"
+import { v16ToV17ErrorEventsAttributeFallbackModule } from "./local-store-migrations/v16-to-v17-error-events-attribute-fallback"
 import type {
 	AnyLocalStoreMigrationModule,
 	LocalStoreMigration,
@@ -125,6 +126,7 @@ export const localStoreMigrations: ReadonlyArray<AnyLocalStoreMigrationModule> =
 	v13ToV14AiTraceIndexModule,
 	v14ToV15CommitShaVcsRevisionModule,
 	v15ToV16AiTraceIndexFilterColumnsModule,
+	v16ToV17ErrorEventsAttributeFallbackModule,
 ]
 
 export const validateMigrationRegistry = (