From d5e677284b59b619ed953e67c2483755996fe685 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 1 Sep 2026 10:51:48 +0200 Subject: [PATCH 1/5] feat(product-events): derive product events from annotated spans MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a fourth feed into `product_events`, alongside browser (session_events MV), server and mobile (POST /v1/events): a span the customer annotated in their own code. span.setAttributes({ "maple.product_event.name": "checkout_completed", "maple.product_event.user_id": user.id, }) The span's other attributes become the event's properties by default, so nothing has to be declared to get a useful event. Two optional controls narrow or replace that: `maple.product_event.include` is a comma-separated allow-list, `maple.product_event.prop.*` are explicit props merged over the base and winning ties, and an EMPTY `include` narrows the base to nothing so the props are all that survive — one mechanism, three tiers, no separate replace flag. Both are span attributes because a materialized view is static SQL per cluster with no per-org config to read. An attribute rather than a UI action because a product event has to be emitted by the code path that performed the thing, at the moment it performed it. Marking a trace by hand marks one sampled trace, cannot be replayed over history, and puts a mutable user-authored row into an append-only fact table. There is no new store and no write path from the dashboard: the span is the record, the product event is its projection. `product_events` gains TraceId/SpanId (DEFAULT '', appended) plus an idx_trace_id bloom filter. Real columns rather than Attributes keys because both directions filter on them, and a Map lookup on this table reads the whole map per row — the cost product_events was split out of session_events to avoid. That column is the link: trace -> its product events productEventsForTraceQuery (trace detail) event -> the traces behind it productEventTraceSamplesQuery (/analytics) Both panels render nothing when empty. Most traces produce no events and a browser track() call has no trace, so silence is the design rather than an empty state on every page. Shipped across all three schema surfaces: ClickHouse migration 0024 (BYO, requiredForIngest: false — the gateway writes neither new column, so ingest routing is not un-readied), product_events_traces_mv for managed orgs, and local schema v13 -> v14. All three backfill the trace half from `traces`, bounded by its 30-day retention against product_events' 365. Known trade, taken deliberately: copying the whole attribute map by default means an annotated span's attributes outlive the span by a factor of twelve, and attribute pickers list the span's full semconv surface until a team sets `include`. Documented in docs/product-events-funnels.md along with the lever. The Attributes expression was executed against ClickHouse 26.2 rather than assumed — mapUpdate, trimBoth and the outer-column lambda capture all behave, and all three tiers produce the intended map. Not in this cut: no MCP tool (an agent cannot walk the link yet) and no SDK helper. Both noted in the doc. --- .../src/routes/internal/query-engine.http.ts | 52 + apps/cli/src/server/local-schema-history.ts | 25 + apps/cli/src/server/local-schema-version.ts | 2 +- apps/cli/src/server/local-store-migrations.ts | 2 + .../v13-to-v14-product-events-from-traces.ts | 451 ++++ apps/cli/src/server/schema-identity.ts | 5 + apps/cli/src/server/schema/local-inserts.json | 2 +- .../src/server/schema/local-schema-v14.sql | 1932 +++++++++++++++++ apps/cli/src/server/schema/local-schema.sql | 57 +- apps/cli/test/local-store-migrations.test.ts | 35 +- apps/cli/test/native-local-store-migration.sh | 2 +- apps/ingest/src/clickhouse_insert_mappings.rs | 8 +- apps/web/src/api/warehouse/product-events.ts | 100 + .../analytics/product-event-trace-samples.tsx | 77 + .../traces/trace-product-events.tsx | 113 + .../services/atoms/warehouse-query-atoms.ts | 16 +- apps/web/src/routes/analytics/index.tsx | 17 + apps/web/src/routes/traces/$traceId.tsx | 22 + docs/product-events-funnels.md | 128 ++ .../0024_product_events_from_traces.ts | 189 ++ .../src/clickhouse/migrations/index.test.ts | 8 +- .../domain/src/clickhouse/migrations/index.ts | 2 + .../domain/src/generated/clickhouse-schema.ts | 7 +- .../generated/tinybird-project-manifest.ts | 11 +- packages/domain/src/http/query-engine.ts | 78 + packages/domain/src/tinybird/datasources.ts | 25 + .../domain/src/tinybird/materializations.ts | 43 +- .../tinybird/product-event-attributes.test.ts | 137 ++ .../src/tinybird/product-event-attributes.ts | 191 ++ .../src/__sql_baseline__/catalog.sql | 38 + .../query-engine/src/ch/builder-fixtures.ts | 23 + packages/query-engine/src/ch/index.ts | 8 + .../src/ch/queries/product-events.ts | 124 ++ packages/query-engine/src/ch/tables.ts | 7 + .../src/registry/product-events.ts | 40 + packages/query-engine/src/registry/queries.ts | 8 +- 36 files changed, 3952 insertions(+), 33 deletions(-) create mode 100644 apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts create mode 100644 apps/cli/src/server/schema/local-schema-v14.sql create mode 100644 apps/web/src/components/analytics/product-event-trace-samples.tsx create mode 100644 apps/web/src/components/traces/trace-product-events.tsx create mode 100644 packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts create mode 100644 packages/domain/src/tinybird/product-event-attributes.test.ts create mode 100644 packages/domain/src/tinybird/product-event-attributes.ts diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 2c2cd8213..7541f05be 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -77,6 +77,8 @@ import { ProductEventsFunnelResponse, ProductEventsFunnelBreakdownResponse, ProductEventNamesResponse, + ProductEventsForTraceResponse, + ProductEventTraceSamplesResponse, CommitSha, FingerprintHash, ServiceName, @@ -211,6 +213,17 @@ const coerceStatusCode = (value: string): StatusCode => // point at fresh traces). Probing the last 48h first prunes to ~2 daily // partitions; only older traces fall back to the unbounded every-partition // probe. +/** + * A ClickHouse `Map(String, String)` as the wire hands it over. Non-string + * values are stringified rather than dropped: the column's own type guarantees + * strings, so anything else means a driver quirk, and losing the key silently + * would be worse than showing its coerced value. + */ +const toStringRecord = (value: unknown): Record => { + if (typeof value !== "object" || value === null || Array.isArray(value)) return {} + return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, String(entry)])) +} + const PROBE_RECENT_WINDOW_MS = 48 * 3_600_000 const toServicePlatformRow = (row: CH.ServicePlatformsOutput) => { @@ -2066,6 +2079,45 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + // Both directions of the trace ↔ product-event link. `Attributes` is + // the one field that is not a scalar: the warehouse hands back a + // Map as an object, and a row that somehow arrives without one + // degrades to `{}` rather than failing the whole panel. + .handle("productEventsForTrace", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventsForTrace, tenant, payload) + return new ProductEventsForTraceResponse({ + data: rows.map((row) => ({ + timestamp: String(row.timestamp), + eventName: String(row.eventName), + spanId: String(row.spanId), + serviceName: String(row.serviceName), + userId: String(row.userId), + groupId: String(row.groupId), + visitorId: String(row.visitorId), + sessionId: String(row.sessionId), + attributes: toStringRecord(row.attributes), + })), + }) + }), + ) + .handle("productEventTraceSamples", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventTraceSamples, tenant, payload) + return new ProductEventTraceSamplesResponse({ + data: rows.map((row) => ({ + traceId: String(row.traceId), + spanId: String(row.spanId), + timestamp: String(row.timestamp), + serviceName: String(row.serviceName), + userId: String(row.userId), + visitorId: String(row.visitorId), + })), + }) + }), + ) .handle("executeRawSql", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 3e1d69283..74af6603a 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -132,4 +132,29 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "353715a6b6c7a05f3227215b072ac95a8bd5ee67d5eec35f8c2b4c86839a1187", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // Product events from traces. `product_events` gains `TraceId`/`SpanId` + // (`DEFAULT ''`, appended) and an `idx_trace_id` bloom filter, and + // `product_events_traces_mv` starts projecting spans annotated with + // `maple.product_event.name` into the table — carrying the span's attribute + // map as the event's properties, narrowable with + // `maple.product_event.include` and overridable with + // `maple.product_event.prop.*`. + // + // The trace half IS backfilled from `traces`, unlike the last two edges: + // there is a source to re-project from. It is bounded by raw traces' + // 30-day retention against `product_events`' 365, so annotated spans older + // than that window are gone and the table accrues them from here. No + // existing row is rewritten — the columns are metadata-only defaults, and + // the browser/server/mobile rows are verified byte-identical afterwards. + // + // projectRevision is carried forward deliberately — it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 14, + fingerprint: "892bcf3b1df69fdd", + digest: "892bcf3b1df69fdd2ca04c738a5f7e21746de2a7e74a2444cf5c4525f9eb4821", + manifestDigest: "4870ae8019002bf1a0b41bba1d6da88366cd72a39e0961e1db24f5cd4dd721d0", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index 0a4391872..af3f256d7 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 13 as const +export const LOCAL_SCHEMA_VERSION = 14 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index 12526a1d3..203e54173 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -49,6 +49,7 @@ import { v9ToV10SemconvKeyRenamesModule } from "./local-store-migrations/v9-to-v import { v10ToV11ProductEventsModule } from "./local-store-migrations/v10-to-v11-product-events" import { v11ToV12ServiceMapEdgeQuantilesModule } from "./local-store-migrations/v11-to-v12-service-map-edge-quantiles" import { v12ToV13ServiceOperationsDiscriminatorsModule } from "./local-store-migrations/v12-to-v13-service-operations-discriminators" +import { v13ToV14ProductEventsFromTracesModule } from "./local-store-migrations/v13-to-v14-product-events-from-traces" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -119,6 +120,7 @@ export const localStoreMigrations: ReadonlyArray = v10ToV11ProductEventsModule, v11ToV12ServiceMapEdgeQuantilesModule, v12ToV13ServiceOperationsDiscriminatorsModule, + v13ToV14ProductEventsFromTracesModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts b/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts new file mode 100644 index 000000000..3e1eb32b5 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts @@ -0,0 +1,451 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cp, mkdir, rm } from "node:fs/promises" +import { dirname, resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V13, + LOCAL_SCHEMA_V13_MANIFEST, + LOCAL_SCHEMA_V13_SQL, + LOCAL_SCHEMA_V14, + LOCAL_SCHEMA_V14_MANIFEST, + LOCAL_SCHEMA_V14_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +const MODULE_ID = "local-0013-to-0014-product-events-from-traces" as const + +/** + * Frozen copy of ClickHouse migration 0024's trace projection. Frozen for the + * reason every edge in this directory freezes its SQL: this module describes one + * step in history, and importing the live projection would silently rewrite what + * v13 -> v14 did the next time it changes. + */ +const PRODUCT_EVENTS_TRACE_COLUMNS = [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + "TraceId", + "SpanId", +].join(", ") + +const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +const PRODUCT_EVENTS_TRACE_FILTER = "SpanAttributes['maple.product_event.name'] != ''" + +/** + * The local mirror of ClickHouse migration 0024. + * + * `product_events` gains `TraceId`/`SpanId` (`DEFAULT ''`) and a bloom filter on + * `TraceId`, and a second view — `product_events_traces_mv` — starts projecting + * spans the user annotated in their own code (`maple.product_event.name`) into + * the table. The trace id is what makes a product event and the trace that + * produced it navigable from either side. + * + * THE TRACE HALF IS BACKFILLED, unlike the last two edges. Every annotated span + * still inside the local store's raw `traces` retention is re-projected, so an + * `EventName` a user just added to their code has history the moment they + * upgrade rather than only going forward — and the backfill is bounded by that + * retention (30 days by default) against `product_events`' own 365, so the older + * part of the window stays empty and accrues from here. + * + * `product_events_mv` — the browser feed — is dropped and recreated too. Its + * SELECT was frozen at 15 columns and the table now has 17; the two new ones + * default to `''`, which is the right value for a browser row, so recreating it + * repairs nothing and instead keeps the view's text and the table's schema + * describing the same thing. + * + * Every statement is idempotent — `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT + * EXISTS`, and a `DELETE WHERE Source = 'trace'` that clears exactly and only + * what the following `INSERT` re-adds — so a resume after a crash between them + * lands in the same place. + */ + +interface V13ToV14State { + readonly module: typeof MODULE_ID + readonly version: 1 + readonly rawRows: Readonly> + readonly productEventRows: ProductEventRowCounts + readonly retentionDays?: number +} + +/** + * What `product_events` held before the edge, and what the backfill is expected + * to add. + * + * `existing` is every row already in the table — at v13 none of them can be a + * trace row, because the source did not exist — and is re-checked as an + * EQUALITY: the backfill must add rows, never disturb one. `expectedTrace` is + * counted from `traces` under the very filter the backfill uses, so verify + * compares against the exact number rather than a lower bound, and an + * `INSERT … SELECT` that half-completed shows up as a mismatch instead of + * passing as "some rows arrived". + */ +interface ProductEventRowCounts { + readonly existing: string + readonly expectedTrace: string +} + +interface V13ToV14Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const isCount = (value: unknown): value is string => typeof value === "string" && /^\d+$/.test(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v13 -> v14 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (!isCount(count)) throw new Error(`v13 -> v14 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v13 -> v14 rawRows contains an unknown table") + return counts +} + +const decodeProductEventRows = (value: unknown): ProductEventRowCounts => { + if (!isRecord(value)) throw new Error("v13 -> v14 productEventRows must be an object") + if (Object.keys(value).some((key) => key !== "existing" && key !== "expectedTrace")) + throw new Error("v13 -> v14 productEventRows contains an unknown field") + if (!isCount(value.existing)) + throw new Error("v13 -> v14 productEventRows.existing must be an unsigned decimal string") + if (!isCount(value.expectedTrace)) + throw new Error("v13 -> v14 productEventRows.expectedTrace must be an unsigned decimal string") + return { existing: value.existing, expectedTrace: value.expectedTrace } +} + +const decodeState = (value: unknown): V13ToV14State => { + if (!isRecord(value)) throw new Error("v13 -> v14 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "productEventRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v13 -> v14 state contains an unknown field") + if (value.module !== MODULE_ID || value.version !== 1) + throw new Error("v13 -> v14 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v13 -> v14 retentionDays must be an integer") + return { + module: MODULE_ID, + version: 1, + rawRows: decodeCounts(value.rawRows), + productEventRows: decodeProductEventRows(value.productEventRows), + ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), + } +} + +const decodeProgress = (value: unknown): V13ToV14Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v13 -> v14 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const scalarCount = (db: Chdb, sql: string): string => { + const rows = parseJsonEachRow<{ count: string }>(db.query(sql)) + const count = rows[0]?.count + if (!isCount(count)) throw new Error(`v13 -> v14 count query returned no row: ${sql}`) + return count +} + +const productEventRowCounts = (db: Chdb): ProductEventRowCounts => ({ + existing: scalarCount(db, "SELECT toString(count()) AS count FROM product_events"), + expectedTrace: scalarCount( + db, + `SELECT toString(count()) AS count FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ), +}) + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V13_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const { rawRows, productEventRows } = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V13_MANIFEST, retentionDays)) + return { rawRows: rawRowCounts(db), productEventRows: productEventRowCounts(db) } + }, + { schemaSql: LOCAL_SCHEMA_V13_SQL, bootstrapSchema: false }, + ) + return { + module: MODULE_ID, + version: 1, + rawRows, + productEventRows, + ...(!(retentionDays === undefined) ? { retentionDays } : undefined), + } +} + +const prepareTarget = async ( + context: MigrationModuleContext, + state: V13ToV14State, +): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await rm(target, { recursive: true, force: true }) + await mkdir(dirname(target), { recursive: true, mode: 0o700 }) + await cp(source, target, { recursive: true, preserveTimestamps: true }) + } + return state +} + +/** + * The columns, the index AND both view drops happen before the v14 bootstrap, in + * the v13-schema block. The ordering is load-bearing in both directions, exactly + * as it was for v12 -> v13: + * + * - `CREATE TABLE IF NOT EXISTS` is a no-op against the cloned store, so the + * v13 snapshot alone leaves `product_events` without its two new columns. + * - `CREATE MATERIALIZED VIEW IF NOT EXISTS` is equally a no-op while the old + * view still exists, and a view's SELECT is frozen at creation. Dropping the + * views AFTER the bootstrap would delete them outright — the bootstrap has + * already skipped them and nothing recreates them. + * + * The backfill then runs in the v14 block, after the bootstrap has created both + * views. It writes `product_events` directly and the views read `session_events` + * and `traces`, so nothing double-fires. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''") + db.exec("ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''") + db.exec( + "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + ) + db.exec("DROP VIEW IF EXISTS product_events_traces_mv") + db.exec("DROP VIEW IF EXISTS product_events_mv") + }, + { schemaSql: LOCAL_SCHEMA_V13_SQL, bootstrapSchema: false }, + ) + return context.openTarget( + (db) => { + db.exec("DELETE FROM product_events WHERE Source = 'trace'") + db.exec( + `INSERT INTO product_events (${PRODUCT_EVENTS_TRACE_COLUMNS}) SELECT ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ) + return { installed: true } as const + }, + { schemaSql: LOCAL_SCHEMA_V14_SQL, bootstrapSchema: true }, + ) +} + +const verify = async ( + context: MigrationModuleContext, + state: V13ToV14State, + _progress: V13ToV14Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V14_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v13 -> v14 raw telemetry verification failed for ${table}`) + } + // Split rather than a single total: an equality on the pre-existing rows + // is what proves the backfill only ADDED, and an equality on the trace + // rows is what proves it added all of them. A total would let one error + // cancel the other out. + const existing = scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source != 'trace'", + ) + if (existing !== state.productEventRows.existing) + throw new Error( + `v13 -> v14 pre-existing product_events row count changed: expected ${state.productEventRows.existing}, found ${existing}`, + ) + const backfilled = scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source = 'trace'", + ) + if (backfilled !== state.productEventRows.expectedTrace) + throw new Error( + `v13 -> v14 backfilled trace product_events row count mismatch: expected ${state.productEventRows.expectedTrace}, found ${backfilled}`, + ) + }, + { schemaSql: LOCAL_SCHEMA_V14_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v13-store", + description: "Clone the stopped v13 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "add-product-event-trace-columns", + description: + "Add TraceId and SpanId to product_events, plus the TraceId bloom filter the trace lookup prunes on", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "backfill-annotated-spans", + description: + "Project every retained span carrying maple.product_event.name into product_events as a Source='trace' row", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "recreate-product-event-views", + description: + "Recreate product_events_mv and create product_events_traces_mv so new rows carry TraceId and annotated spans keep arriving", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v14-schema", + description: + "Verify the v14 physical schema, retained raw telemetry counts, and that the backfill added exactly the annotated spans and disturbed no existing row", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v13 store is cloned byte-for-byte before any DDL runs.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "Read-only source of the backfill; the row count is verified unchanged alongside every other raw telemetry table.", + }, + { + name: "product_events (browser, server and mobile rows)", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "Two columns are added as metadata-only defaults, no part is rewritten, and the count of rows whose Source is not 'trace' is verified byte-identical after the backfill.", + }, + { + // Fully rebuilt within the raw window, then accrued: unlike the last two + // edges there IS a source to re-project from, so the shorter retention is + // the only bound. + name: "product_events (trace rows)", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Every annotated span still inside raw traces retention is re-projected, and the resulting row count is verified to equal the count of matching spans. Annotated spans older than that window are gone from traces and cannot be rebuilt; the table accrues them from the migration forward.", + preservationInterval: "the raw traces retention window", + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, +] + +export const v13ToV14ProductEventsFromTracesModule: LocalStoreMigrationModule< + V13ToV14State, + V13ToV14Progress +> = { + id: MODULE_ID, + moduleVersion: 1, + description: + "Add TraceId/SpanId to product_events and project spans annotated with maple.product_event.name into it, backfilled from retained traces", + from: LOCAL_SCHEMA_V13, + to: LOCAL_SCHEMA_V14, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index f88283be5..57734100e 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -12,6 +12,7 @@ import schemaV10Sql from "./schema/local-schema-v10.sql" with { type: "text" } import schemaV11Sql from "./schema/local-schema-v11.sql" with { type: "text" } import schemaV12Sql from "./schema/local-schema-v12.sql" with { type: "text" } import schemaV13Sql from "./schema/local-schema-v13.sql" with { type: "text" } +import schemaV14Sql from "./schema/local-schema-v14.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -69,6 +70,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV11Sql, schemaV12Sql, schemaV13Sql, + schemaV14Sql, ] export interface LocalSchemaSnapshot { @@ -119,6 +121,8 @@ export const LOCAL_SCHEMA_V12_SQL = snapshotAt(12).sql export const LOCAL_SCHEMA_V12_MANIFEST = snapshotAt(12).manifest export const LOCAL_SCHEMA_V13_SQL = snapshotAt(13).sql export const LOCAL_SCHEMA_V13_MANIFEST = snapshotAt(13).manifest +export const LOCAL_SCHEMA_V14_SQL = snapshotAt(14).sql +export const LOCAL_SCHEMA_V14_MANIFEST = snapshotAt(14).manifest export interface LocalSchemaIdentity { readonly version: number @@ -162,6 +166,7 @@ export const LOCAL_SCHEMA_V10 = identityAt(10) export const LOCAL_SCHEMA_V11 = identityAt(11) export const LOCAL_SCHEMA_V12 = identityAt(12) export const LOCAL_SCHEMA_V13 = identityAt(13) +export const LOCAL_SCHEMA_V14 = identityAt(14) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 3956676cb..ec2ae3821 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "27390bcb1160c0e9f9c1f91aba7c955e5b52a56dda7a7c0bfc6182ef07eaa945", + "projectRevision": "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v14.sql b/apps/cli/src/server/schema/local-schema-v14.sql new file mode 100644 index 000000000..c8cab6145 --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v14.sql @@ -0,0 +1,1932 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb +-- localSchemaVersion: 14 + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['deployment.commit_sha'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 313e1e2ef..c8cab6145 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 27390bcb1160c0e9f9c1f91aba7c955e5b52a56dda7a7c0bfc6182ef07eaa945 --- localSchemaVersion: 13 +-- projectRevision: 09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb +-- localSchemaVersion: 14 CREATE TABLE IF NOT EXISTS alert_checks ( OrgId LowCardinality(String), @@ -346,8 +346,11 @@ CREATE TABLE IF NOT EXISTS product_events ( Url String DEFAULT '', ServiceName LowCardinality(String) DEFAULT '', Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, - INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -1325,10 +1328,56 @@ SELECT path(Url) AS PagePath, Url, '' AS ServiceName, - Attributes + Attributes, + '' AS TraceId, + '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom'); +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS SELECT OrgId, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index 1697abd0b..e08558d20 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -25,6 +25,7 @@ import { LOCAL_SCHEMA_V11, LOCAL_SCHEMA_V12, LOCAL_SCHEMA_V13, + LOCAL_SCHEMA_V14, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -68,16 +69,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v12 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("7c44772116706420") - expect(SCHEMA_DIGEST).toBe("7c4477211670642086313b71593d848cbadefc24142a1c6e0fe5fd93a8dd7a6e") + it("matches the generated v14 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("892bcf3b1df69fdd") + expect(SCHEMA_DIGEST).toBe("892bcf3b1df69fdd2ca04c738a5f7e21746de2a7e74a2444cf5c4525f9eb4821") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(13) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V13) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(14) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V14) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -159,6 +160,7 @@ describe("current local schema identity", () => { "identity_links_mv", "product_events", "product_events_mv", + "product_events_traces_mv", ]) const errorEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "error_events_mv", @@ -185,7 +187,7 @@ describe("current local schema identity", () => { expect(productEvents?.engine).toBe("MergeTree") expect(productEvents?.orderBy).toBe("(OrgId, Timestamp, VisitorId, SessionId, Seq)") expect(productEvents?.ttl).toContain("365 DAY") - expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id"]) + expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id", "idx_trace_id"]) expect(productEvents?.columns.map((column) => column.name)).toEqual([ "OrgId", "Timestamp", @@ -202,6 +204,10 @@ describe("current local schema identity", () => { "Url", "ServiceName", "Attributes", + // Appended, not inserted: `ALTER TABLE … ADD COLUMN` puts them last, and + // every projection into this table has to match that order. + "TraceId", + "SpanId", ]) const productEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "product_events_mv", @@ -230,14 +236,21 @@ describe("current local schema identity", () => { "GroupId", ]) const v10Names = new Set(LOCAL_SCHEMA_V10_MANIFEST.objects.map((object) => object.name)) - const v11Names = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) - expect([...v11Names].filter((name) => !v10Names.has(name))).toEqual([ + // Named for what it is: the CURRENT manifest, not v11's. The delta below is + // therefore everything since v10, which is why v14's + // `product_events_traces_mv` belongs in it. + const currentObjectNames = new Set(LOCAL_SCHEMA_MANIFEST.objects.map((object) => object.name)) + expect([...currentObjectNames].filter((name) => !v10Names.has(name))).toEqual([ "identity_links", "identity_links_mv", "product_events", "product_events_mv", + "product_events_traces_mv", + ]) + expect([...v10Names].filter((name) => !currentObjectNames.has(name))).toEqual([ + "web_events", + "web_events_mv", ]) - expect([...v10Names].filter((name) => !v11Names.has(name))).toEqual(["web_events", "web_events_mv"]) }) it("recognises Apple crash frames at v8 but not before", () => { @@ -270,6 +283,7 @@ describe("local migration registry", () => { "local-0010-to-0011-product-events", "local-0011-to-0012-service-map-edge-quantiles", "local-0012-to-0013-service-operations-discriminators", + "local-0013-to-0014-product-events-from-traces", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -316,7 +330,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 14, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 15, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -1318,6 +1332,7 @@ describe("v10 -> v11 product events module", () => { "local-0010-to-0011-product-events", "local-0011-to-0012-service-map-edge-quantiles", "local-0012-to-0013-service-operations-discriminators", + "local-0013-to-0014-product-events-from-traces", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 3f72e7fe8..2d829892c 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 13 and .schema == "7c44772116706420"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 14 and .schema == "892bcf3b1df69fdd"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 697512d2e..d1cf2cfda 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "27390bcb1160c0e9f9c1f91aba7c955e5b52a56dda7a7c0bfc6182ef07eaa945"; +pub const PROJECT_REVISION: &str = "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse @@ -85,9 +85,9 @@ pub const DATASOURCES: &[InsertMapping] = &[ InsertMapping { datasource: "product_events", table: "product_events", - columns: &["OrgId", "Timestamp", "Source", "SessionId", "Seq", "VisitorId", "UserId", "GroupId", "Kind", "EventName", "Host", "PagePath", "Url", "ServiceName", "Attributes"], - selects: &["__ORG__", "timestamp", "source", "session_id", "seq", "visitor_id", "user_id", "group_id", "kind", "event_name", "host", "page_path", "url", "service_name", "attributes"], - input_schema: "timestamp DateTime64(9), source LowCardinality(String), session_id String, seq UInt32, visitor_id String, user_id String, group_id String, kind LowCardinality(String), event_name String, host LowCardinality(String), page_path String, url String, service_name LowCardinality(String), attributes Map(String, String)", + columns: &["OrgId", "Timestamp", "Source", "SessionId", "Seq", "VisitorId", "UserId", "GroupId", "Kind", "EventName", "Host", "PagePath", "Url", "ServiceName", "Attributes", "TraceId", "SpanId"], + selects: &["__ORG__", "timestamp", "source", "session_id", "seq", "visitor_id", "user_id", "group_id", "kind", "event_name", "host", "page_path", "url", "service_name", "attributes", "trace_id", "span_id"], + input_schema: "timestamp DateTime64(9), source LowCardinality(String), session_id String, seq UInt32, visitor_id String, user_id String, group_id String, kind LowCardinality(String), event_name String, host LowCardinality(String), page_path String, url String, service_name LowCardinality(String), attributes Map(String, String), trace_id String, span_id String", }, ]; diff --git a/apps/web/src/api/warehouse/product-events.ts b/apps/web/src/api/warehouse/product-events.ts index 687d2b718..79a93b770 100644 --- a/apps/web/src/api/warehouse/product-events.ts +++ b/apps/web/src/api/warehouse/product-events.ts @@ -5,10 +5,13 @@ // `keyBy`, the session step, and the breakdown grouping. import { Effect, Schema } from "effect" +import { TraceId } from "@maple/domain" import { ProductEventNamesRequest, + ProductEventsForTraceRequest, ProductEventsFunnelBreakdownRequest, ProductEventsFunnelRequest, + ProductEventTraceSamplesRequest, } from "@maple/domain/http" import { FUNNEL_WIDGET_BREAKDOWN_LIMIT, @@ -63,6 +66,103 @@ const getProductEventNamesEffect = Effect.fn("QueryEngine.getProductEventNames") return { data: result.data satisfies ReadonlyArray } }) +// Trace ↔ product event. +// +// A span annotated in the customer's own code with `maple.product_event.name` +// becomes a `product_events` row carrying its `TraceId`, and these two read that +// column from either end: the trace view lists what a request accomplished, and +// an event name lists the requests that accomplished it. + +// `TraceId`, not a plain string: `decodeInput` is the boundary that turns a +// malformed id in the URL into a decode failure here rather than a warehouse +// error four hops later. +const ProductEventsForTraceInputSchema = Schema.Struct({ + ...TimeWindowFields, + traceId: TraceId, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventsForTraceInput = (typeof ProductEventsForTraceInputSchema)["Encoded"] + +export interface TraceProductEvent { + timestamp: string + eventName: string + /** The annotated span within the trace — deep-links to it in the waterfall. */ + spanId: string + serviceName: string + userId: string + groupId: string + visitorId: string + sessionId: string + /** `maple.product_event.prop.*` attributes, prefix stripped. */ + attributes: Record +} + +export function getProductEventsForTrace({ data }: { data: GetProductEventsForTraceInput }) { + return getProductEventsForTraceEffect({ data }) +} + +const getProductEventsForTraceEffect = Effect.fn("QueryEngine.getProductEventsForTrace")(function* ({ + data, +}: { + data: GetProductEventsForTraceInput +}) { + const input = yield* decodeInput(ProductEventsForTraceInputSchema, data, "getProductEventsForTrace") + + const result = yield* runWarehouseQuery("productEventsForTrace", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventsForTrace({ + payload: new ProductEventsForTraceRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } +}) + +const ProductEventTraceSamplesInputSchema = Schema.Struct({ + ...TimeWindowFields, + eventName: Schema.String, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventTraceSamplesInput = (typeof ProductEventTraceSamplesInputSchema)["Encoded"] + +export interface ProductEventTraceSample { + traceId: string + spanId: string + timestamp: string + serviceName: string + userId: string + visitorId: string +} + +export function getProductEventTraceSamples({ data }: { data: GetProductEventTraceSamplesInput }) { + return getProductEventTraceSamplesEffect({ data }) +} + +const getProductEventTraceSamplesEffect = Effect.fn("QueryEngine.getProductEventTraceSamples")( + function* ({ data }: { data: GetProductEventTraceSamplesInput }) { + const input = yield* decodeInput( + ProductEventTraceSamplesInputSchema, + data, + "getProductEventTraceSamples", + ) + + const result = yield* runWarehouseQuery("productEventTraceSamples", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventTraceSamples({ + payload: new ProductEventTraceSamplesRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } + }, +) + // Dashboard funnel widget (route data source `product_events_funnel`). // // The widget's stored `display.funnel` definition — steps, key, window, an diff --git a/apps/web/src/components/analytics/product-event-trace-samples.tsx b/apps/web/src/components/analytics/product-event-trace-samples.tsx new file mode 100644 index 000000000..f96ec8a66 --- /dev/null +++ b/apps/web/src/components/analytics/product-event-trace-samples.tsx @@ -0,0 +1,77 @@ +import { Link } from "@tanstack/react-router" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { productEventTraceSamplesResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { formatTimestampInTimezone } from "@/lib/timezone-format" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" +import { ChartBarTrendUpIcon } from "@/components/icons" + +/** + * Recent traces behind one product event — the other half of the link a + * `maple.product_event.name` span attribute creates. + * + * Only events annotated on a span can answer this. A `track()` call from the + * browser and a `POST /v1/events` row carry no trace, so for those the list is + * empty and this renders nothing rather than an empty state: "no traces" is not + * a finding about the event, it just means the event was not emitted from a + * span, and saying so on every browser event would be noise on the majority of + * them. + */ +export function ProductEventTraceSamples({ + eventName, + startTime, + endTime, +}: { + eventName: string + startTime: string + endTime: string +}) { + const { effectiveTimezone } = useTimezonePreference() + const result = useAtomValue( + productEventTraceSamplesResultAtom({ data: { eventName, startTime, endTime, limit: 10 } }), + ) + + return Result.builder(result) + .onSuccess((response) => { + if (response.data.length === 0) return null + return ( +
+
+ +

Traces behind “{eventName}”

+
+
    + {response.data.map((sample) => ( +
  • + + + {sample.traceId.slice(0, 8)} + + {sample.serviceName === "" ? null : {sample.serviceName}} + {sample.userId || sample.visitorId ? ( + + · {sample.userId || sample.visitorId} + + ) : null} + + {formatTimestampInTimezone(sample.timestamp, { + timeZone: effectiveTimezone, + })} + + +
    • + ))} +
+
+ ) + }) + .onError(() => null) + .orElse(() => null) +} diff --git a/apps/web/src/components/traces/trace-product-events.tsx b/apps/web/src/components/traces/trace-product-events.tsx new file mode 100644 index 000000000..71c14e61d --- /dev/null +++ b/apps/web/src/components/traces/trace-product-events.tsx @@ -0,0 +1,113 @@ +import { formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { productEventsForTraceResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { ChartBarTrendUpIcon } from "@/components/icons" +import { Badge } from "@maple/ui/components/ui/badge" +import type { TraceProductEvent } from "@/api/warehouse/product-events" + +/** Same margin and reasoning as `TraceLogsLink`: clock skew between services can + * stamp an annotated span slightly outside the root span's own window, and the + * bound is also what keeps the lookup off every retained partition. */ +const WINDOW_MARGIN_MS = 5 * 60 * 1000 + +/** + * The product events this trace produced — spans the team annotated in their own + * code with `maple.product_event.name`. + * + * Renders nothing while loading, on failure, and when the trace produced none, + * which is the overwhelming majority of traces. That silence is the whole design + * of the panel: it sits unconditionally in the trace body and only appears on + * the traces where a request actually accomplished something the business + * counts, so it reads as a finding rather than as another empty section. + * + * Clicking an event selects its span in the waterfall, which is the point of the + * link — "the conversion happened, and here is the code path that did it." + */ +export function TraceProductEvents({ + traceId, + traceStartTime, + totalDurationMs, + onSelectSpan, +}: { + traceId: string + traceStartTime: string + totalDurationMs: number + onSelectSpan: (spanId: string) => void +}) { + const traceStartMs = parseWarehouseDateTime(traceStartTime) + const window = Number.isNaN(traceStartMs) + ? undefined + : { + startTime: formatWarehouseDateTime(traceStartMs - WINDOW_MARGIN_MS), + endTime: formatWarehouseDateTime(traceStartMs + totalDurationMs + WINDOW_MARGIN_MS), + } + + const result = useAtomValue( + productEventsForTraceResultAtom({ + data: { traceId, startTime: window?.startTime ?? "", endTime: window?.endTime ?? "" }, + }), + ) + + if (window === undefined) return null + + return Result.builder(result) + .onSuccess((response) => { + if (response.data.length === 0) return null + return ( +
+
+ +

Product events

+ {response.data.length} +
+
    + {response.data.map((event) => ( + + ))} +
+
+ ) + }) + .onError(() => null) + .orElse(() => null) +} + +function ProductEventRow({ + event, + onSelectSpan, +}: { + event: TraceProductEvent + onSelectSpan: (spanId: string) => void +}) { + // UserId first, then GroupId, then VisitorId — the same precedence the funnel + // person key uses, so the identity shown here is the one the event will be + // counted under rather than whichever field happened to be set. + const person = event.userId || event.groupId || event.visitorId + const props = Object.entries(event.attributes) + + return ( +
  • + +
    • + ) +} diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index b3023b3cf..de9d9ee2d 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -123,7 +123,11 @@ import { getWebAnalyticsSummary, getWebAnalyticsTimeseries, } from "@/api/warehouse/web-analytics" -import { getProductEventNames } from "@/api/warehouse/product-events" +import { + getProductEventNames, + getProductEventsForTrace, + getProductEventTraceSamples, +} from "@/api/warehouse/product-events" /** * The error union every warehouse server function fails with: the structured @@ -385,6 +389,16 @@ export const productEventNamesResultAtom = makeQueryAtomFamily(getProductEventNa staleTime: 60_000, }) +// A completed trace's product events never change, so this is only ever refetched +// because the trace is still open. 60s matches the route cache behind it. +export const productEventsForTraceResultAtom = makeQueryAtomFamily(getProductEventsForTrace, { + staleTime: 60_000, +}) + +export const productEventTraceSamplesResultAtom = makeQueryAtomFamily(getProductEventTraceSamples, { + staleTime: 60_000, +}) + export const getReplayResultAtom = makeQueryAtomFamily(getReplay, { staleTime: 60_000, }) diff --git a/apps/web/src/routes/analytics/index.tsx b/apps/web/src/routes/analytics/index.tsx index a95d86894..f24e2d6b6 100644 --- a/apps/web/src/routes/analytics/index.tsx +++ b/apps/web/src/routes/analytics/index.tsx @@ -19,6 +19,7 @@ import { type BreakdownDimension, } from "@/components/analytics/analytics-breakdown-panel" import { AnalyticsBotNotice } from "@/components/analytics/analytics-bot-notice" +import { ProductEventTraceSamples } from "@/components/analytics/product-event-trace-samples" import { AnalyticsFilterSidebar } from "@/components/analytics/analytics-filter-sidebar" import { AnalyticsLiveBadge } from "@/components/analytics/analytics-live-badge" import { @@ -588,6 +589,13 @@ function AnalyticsContent({ { id: "events", dimensions: eventDimensions, wide: true }, ] + // The reverse of the trace view's product-events panel, shown at the + // one moment it is asked for: the user has filtered to a single event + // and now wants to see requests that fired it. Renders nothing unless + // the event came from an annotated span, so browser `track()` events + // simply do not grow a section. + const eventName = filters.eventName + return (
    {cards.map((card) => ( @@ -603,6 +611,15 @@ function AnalyticsContent({ />
    ))} + {eventName === undefined ? null : ( +
    + +
    + )} ) }) diff --git a/apps/web/src/routes/traces/$traceId.tsx b/apps/web/src/routes/traces/$traceId.tsx index 0fa679af2..beadf13a8 100644 --- a/apps/web/src/routes/traces/$traceId.tsx +++ b/apps/web/src/routes/traces/$traceId.tsx @@ -13,6 +13,7 @@ import { QueryErrorState } from "@/components/common/query-error-state" import { TraceViewTabs } from "@maple/ui/components/traces/trace-view-tabs" import { SpanDetailPanel } from "@/components/traces/span-detail-panel" import { TraceAnatomyStrip } from "@/components/traces/trace-anatomy-strip" +import { TraceProductEvents } from "@/components/traces/trace-product-events" import { Skeleton } from "@maple/ui/components/ui/skeleton" import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "@maple/ui/components/ui/resizable" import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@maple/ui/components/ui/sheet" @@ -245,6 +246,20 @@ function TraceDetailContent({ [search.spanId, navigate], ) + // The product-events panel knows a span ID, not a `SpanNode` — its rows come + // from `product_events`, not the hierarchy. Selecting by id keeps the two + // sources from having to agree on a node shape. + const handleSelectSpanId = React.useCallback( + (spanId: string) => { + if (search.spanId === spanId) return + navigate({ + search: (prev: Record) => ({ ...prev, spanId }), + replace: true, + }) + }, + [search.spanId, navigate], + ) + const handleCloseSpanDetails = React.useCallback(() => { navigate({ search: (prev: Record) => ({ ...prev, spanId: undefined }), @@ -335,6 +350,13 @@ function TraceDetailContent({ commitSha={commitSha} /> + + {isMobile ? ( // A 60/40 side-by-side split leaves each pane ~150px on a phone. Give the waterfall // the full width and float the span detail over it instead. diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index fc1e135f0..b661466e8 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -277,3 +277,131 @@ the `product_events_funnel` route params (`ProductEventsFunnelWidgetParams` in ` which the browser server function and the share API's route plan both decode. With a breakdown the route answers `{ name, value, group }` rows (top 6 groups by step-1 count) and the funnel chart draws one bar per group per step with a legend. + +## Product events from traces — annotate in code (2026-09) + +The fourth feed into `product_events`, after browser (`session_events` MV), server and mobile +(`POST /v1/events`). A team marks a span they already emit and it becomes a funnel step that +links back to the request that performed it. + +```ts +span.setAttributes({ + "maple.product_event.name": "checkout_completed", // required — presence is the predicate + "maple.product_event.user_id": user.id, // optional identity + "maple.product_event.group_id": org.id, + "maple.product_event.visitor_id": anonId, + "maple.product_event.url": req.url, // optional page context +}) +``` + +**Every other attribute on the span becomes an event property by default.** Nothing has to be +declared to get started — whatever the team already sets on the span (`plan`, `order.total`, the +full HTTP/DB semconv surface) lands in `Attributes` and is available to funnel breakdowns. The +`maple.product_event.*` control keys are stripped, since they are already promoted to their own +columns. + +Two optional controls narrow or replace that default. Both are themselves span attributes: a +materialized view is static SQL per cluster and has no per-org config to read. + +```ts +"maple.product_event.include": "plan,seats" // ONLY these span keys (whitespace trimmed) +"maple.product_event.prop.plan": "pro" // explicit prop, merged over the base, wins ties +"maple.product_event.include": "" // and together: full overwrite +``` + +Three tiers out of one mechanism rather than three modes to pick between: + +| `include` | `prop.*` | `Attributes` | +| --- | --- | --- | +| absent | — | every span attribute | +| absent | set | every span attribute, with the props overriding on a key collision | +| `"plan,seats"` | — | only `plan` and `seats` | +| `""` | set | only the props — the overwrite case | + +`include` switches on **key presence**, not on a non-empty value, which is what makes the empty +string mean "no span attributes" rather than "no filter". There is no separate replace flag to get +wrong, and `mapUpdate(base, props)` argument order is the override rule — swapped, an override +would be discarded exactly when the key it meant to correct was already present. + +Verified against ClickHouse 26.2, all three tiers: + +| Scenario | Span attributes | Result | +| --- | --- | --- | +| default | `http.method`, `plan=free`, `seats=5`, `prop.plan=pro` | `{http.method, seats, plan:'pro'}` | +| `include: "plan, seats"` | + `noise` | `{plan:'free', seats:'5'}` | +| `include: ""` + `prop.plan=pro` | `http.method`, `plan=free` | `{plan:'pro'}` | + +The contract lives in one place — `packages/domain/src/tinybird/product-event-attributes.ts` — +and is read by exactly two consumers that must agree byte for byte: `productEventsTracesMv` +(managed orgs, via `tinybird deploy`) and the frozen copy inside ClickHouse migration 0024 (BYO +clusters). The migration's copy is deliberately NOT imported from the constant: a delta migration +describes one step in history, and a shared constant would silently rewrite what 0024 did the next +time the live projection changes. + +### Why an attribute and not a UI action + +A product event has to be emitted by the code path that performed the thing, at the moment it +performed it. Marking a trace by hand in the UI marks *one sampled trace*, cannot be replayed over +history, and puts a mutable user-authored row into an append-only fact table. An attribute marks +every trace the path produces, applies retroactively across the whole `traces` retention window, +and is reviewable in the customer's own diff. There is no second store and no write path from the +dashboard — the span is the record, the product event is its projection. + +### The link + +`product_events` gained `TraceId`/`SpanId` (migration 0024, `DEFAULT ''`, appended, plus a +`bloom_filter` on `TraceId`). Non-empty only on `Source = 'trace'` rows. Real columns rather than +`Attributes` keys because both directions filter on them, and a `Map` lookup on this table reads +the whole map per row — the exact cost `product_events` was split out of `session_events` to avoid. + +| Direction | Query | Surface | +| --- | --- | --- | +| trace → its product events | `productEventsForTraceQuery` | trace detail page, under the anatomy strip | +| event → the traces behind it | `productEventTraceSamplesQuery` | `/analytics`, when the `eventName` filter is set | + +Both are `profile: "list"` with a flat `cache: 60` rather than `timeRangeCache`: they are point +lookups whose answer does not widen with the range asked about, and a completed trace's events +never change at all. + +### What it costs + +The MV predicate is one `Map` value read per incoming span, on the same block every other `traces` +MV already fires on. An MV sees the insert block, not the table, so `idx_span_attr_keys` does not +help it — this is a real, deliberately small per-span ingest cost. + +Copying the whole `SpanAttributes` map **by default** is the deliberate expensive choice. A server +span's map is dominated by HTTP/DB semconv keys, and `product_events` keeps 365 days against raw +`traces`' 30 — so an annotated span's attributes outlive the span itself by a factor of twelve. What +the default buys is that nothing has to be declared to get a useful event; `include` is the lever +for a team that has measured the cost and wants it back, and it is a one-line change on the span +rather than a schema migration. + +The practical consequence to watch: attribute pickers over product events list the span's whole +semconv surface for any team that has not set `include`. If that becomes the dominant cost across +orgs rather than for one of them, the lever is a per-org key denylist at the MV — the per-span +`include` handles the single-team case already. + +The whole `Attributes` expression only evaluates for rows passing the `WHERE`, i.e. annotated spans, +so its cost is paid per product event rather than per span. The predicate itself stays one map +lookup. + +### Rollout + +1. **Managed**: `bun run --cwd apps/api tinybird:deploy` creates `product_events_traces_mv` and + adds the two columns, then an explicit `tb` populate from `traces` (bounded by its 30-day TTL). + Blocked on the same manual step the rest of this document's checklist is — see + `project_product_events_tinybird_rollout_pending`. +2. **BYO ClickHouse**: migration 0024, `requiredForIngest: false` — the gateway writes neither new + column, so ingest routing is not un-readied. Backfills the trace half from `traces` itself. +3. **Local CLI**: local schema v13 → v14, same backfill. + +### Not in this cut + +- **No MCP tool.** `list_product_events` still returns names only, and `inspect_trace` does not + surface a trace's product events. An agent cannot walk the link yet; the queries and the HTTP + routes it would sit on both exist. +- **No SDK helper.** Teams set the attributes by hand on whatever span API they already use. A + `markProductEvent(span, name, { props, include })` in `@maple-dev/effect-sdk` would be the obvious + next step — it is a wrapper over `setAttributes` that builds the `prop.*` keys and joins + `include`, not new machinery, and it is where the empty-string overwrite idiom would get a name + (`attributes: "none"`) instead of being a documented convention. diff --git a/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts b/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts new file mode 100644 index 000000000..f47cefee0 --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts @@ -0,0 +1,189 @@ +import type { BackfillSpec } from "../backfill" + +/** + * Frozen copy of the trace→product-event projection as of this migration. + * + * NOT imported from `tinybird/product-event-attributes.ts`, deliberately, and + * for the reason migration 0019 spells out: a delta migration describes one step + * in history. A shared constant would silently rewrite what this migration did + * the next time the live projection changes, and a BYO cluster replaying the + * chain would land somewhere the chain never says it goes. + * + * Shared *within* this file between the materialized view and the backfill, + * which is the 0021 pattern — two copies of one SELECT is two chances for a + * backfilled span and a live span to disagree. + */ +const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +const PRODUCT_EVENTS_TRACE_FILTER = "SpanAttributes['maple.product_event.name'] != ''" + +/** + * Backfill of the trace half of {@link migration_0024_product_events_from_traces}. + * + * Row-wise, so any chunk boundary is safe. Idempotent by `DELETE WHERE Source = + * 'trace'` before it runs — same shape as 0021's browser half, and safe for the + * same reason: it clears only rows this projection owns, never a browser row + * from the other view or a directly ingested one that has no source to come back + * from. + * + * Bounded by `traces`' own 30-day TTL, which is all there is to rebuild from. + * `product_events` keeps 365 days, so an org that annotates spans today sees + * ~30 days of history immediately and accrues the rest going forward. + */ +export const productEventsTracesBackfill: BackfillSpec = { + kind: "backfill", + target: "product_events", + columns: [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + "TraceId", + "SpanId", + ], + from: "traces", + tsColumn: "Timestamp", + select: PRODUCT_EVENTS_TRACE_PROJECTION_SQL, + where: PRODUCT_EVENTS_TRACE_FILTER, +} + +/** + * Migration 0024 — product events annotated in code. + * + * A customer marks a span they already emit: + * + * ``` + * span.setAttributes({ "maple.product_event.name": "checkout_completed" }) + * ``` + * + * and it becomes a row in `product_events` that a funnel steps on, carrying the + * `TraceId` it came from. The annotation is instrumentation, not a UI action and + * not a second store: the span remains the record and the product event is its + * projection, so it applies to every trace the annotated code path produces + * rather than to the one trace someone happened to be looking at. + * + * Two things, in order: + * + * 1. `product_events` gains `TraceId`/`SpanId` (`DEFAULT ''`) plus a bloom + * filter on `TraceId`. Metadata-only `ADD COLUMN`s, appended, which is why + * they sit last in the schema and last in every projection. + * 2. `product_events_traces_mv` projects annotated spans in, and the trace half + * is backfilled from `traces`' 30-day window. + * + * `product_events_mv` is dropped and re-created rather than left alone: an MV's + * SELECT is fixed at creation, so the pre-0024 body writes 15 columns into a + * 17-column table and every browser row inserted between the ALTER and the + * re-create would take defaults for the two new columns. They default to `''`, + * which is the correct value for a browser row — so this is ordering hygiene + * rather than a repair, and the re-create is what keeps the view's text and the + * table's schema describing the same thing. + * + * Re-runnable by construction: `ADD COLUMN IF NOT EXISTS`, views dropped before + * the backfill, and `DELETE WHERE Source = 'trace'` clears exactly and only what + * the backfill is about to re-insert. + * + * **BYO ClickHouse only.** Managed orgs get the same view via `tinybird deploy` + * from `materializations.ts`, with the populate as an explicit `tb` step at + * deploy time (see 0014 and 0021 — the SDK has no populate option). + * + * `requiredForIngest: false`. The two new columns are MV- and backfill-written; + * the Rust gateway's `/v1/events` path does not emit them, and a cluster without + * them still accepts every row the gateway sends. Bumping + * `clickHouseSchemaVersion` would un-ready ingest routing for every BYO-CH org + * over a feature none of their existing writers touch. + */ +export const migration_0024_product_events_from_traces = { + version: 24, + description: + "Add TraceId/SpanId to product_events and materialize product events from spans carrying the maple.product_event.name attribute", + requiredForIngest: false, + statements: [ + "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''", + "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''", + "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + "DROP VIEW IF EXISTS product_events_traces_mv", + "DROP VIEW IF EXISTS product_events_mv", + // Idempotency for the trace half only — never touches a browser row or a + // directly ingested one. + "DELETE FROM product_events WHERE Source = 'trace'", + productEventsTracesBackfill, + // Frozen copy of the browser projection as of 0021, plus the two new + // columns explicitly at `''`. Same freezing rule as above: this is what + // the view's body was at this point in history. + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId +FROM session_events +WHERE Type IN ('navigation', 'custom')`, + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT +${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} +FROM traces +WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 22967be77..95bedb548 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -28,6 +28,7 @@ import { migration_0019_mv_sweep } from "./0019_mv_sweep" import { migration_0020_semconv_key_renames } from "./0020_semconv_key_renames" import { migration_0022_service_map_edge_quantiles } from "./0022_service_map_edge_quantiles" import { migration_0023_service_operations_discriminators } from "./0023_service_operations_discriminators" +import { migration_0024_product_events_from_traces } from "./0024_product_events_from_traces" import { migration_0021_product_events } from "./0021_product_events" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" @@ -44,10 +45,10 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, ]) - expect(migrations.at(-1)).toBe(migration_0023_service_operations_discriminators) - expect(latestMigrationVersion).toBe(23) + expect(migrations.at(-1)).toBe(migration_0024_product_events_from_traces) + expect(latestMigrationVersion).toBe(24) // 0010 and 0014-0020 are read-path only and skipped by the ingest-gating // version; 0021 is not — the gateway writes `session_events`' new identity // columns and `product_events` directly, so a BYO-CH org must apply it @@ -66,6 +67,7 @@ describe("ClickHouse migrations", () => { expect(migration_0020_semconv_key_renames.requiredForIngest).toBe(false) expect(migration_0022_service_map_edge_quantiles.requiredForIngest).toBe(false) expect(migration_0023_service_operations_discriminators.requiredForIngest).toBe(false) + expect(migration_0024_product_events_from_traces.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index b8e3cfee8..3738b43a9 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -22,6 +22,7 @@ import { migration_0020_semconv_key_renames } from "./0020_semconv_key_renames" import { migration_0021_product_events } from "./0021_product_events" import { migration_0022_service_map_edge_quantiles } from "./0022_service_map_edge_quantiles" import { migration_0023_service_operations_discriminators } from "./0023_service_operations_discriminators" +import { migration_0024_product_events_from_traces } from "./0024_product_events_from_traces" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -76,6 +77,7 @@ export const migrations: ReadonlyArray = [ migration_0021_product_events, migration_0022_service_map_edge_quantiles, migration_0023_service_operations_discriminators, + migration_0024_product_events_from_traces, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 56e8ed20b..a001e1164 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "27390bcb1160c0e9f9c1f91aba7c955e5b52a56dda7a7c0bfc6182ef07eaa945" as const +export const projectRevision = "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", @@ -18,7 +18,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS metrics_gauge (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n BucketCounts Array(UInt64),\n ExplicitBounds Array(Float64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_sum (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n AggregationTemporality Int32,\n IsMonotonic Bool\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n TraceId String DEFAULT '',\n SpanId String DEFAULT '',\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4,\n INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_external_edges_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_map_children (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, ParentSpanId, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", @@ -55,7 +55,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", - "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes,\n '' AS TraceId,\n '' AS SpanId\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'trace' AS Source,\n SpanAttributes['session.id'] AS SessionId,\n 0 AS Seq,\n SpanAttributes['maple.product_event.visitor_id'] AS VisitorId,\n SpanAttributes['maple.product_event.user_id'] AS UserId,\n SpanAttributes['maple.product_event.group_id'] AS GroupId,\n 'custom' AS Kind,\n SpanAttributes['maple.product_event.name'] AS EventName,\n domain(SpanAttributes['maple.product_event.url']) AS Host,\n path(SpanAttributes['maple.product_event.url']) AS PagePath,\n SpanAttributes['maple.product_event.url'] AS Url,\n ServiceName,\n mapUpdate(\n CAST(\n mapFilter(\n (k, v) -> NOT startsWith(k, 'maple.product_event.')\n AND (\n NOT has(mapKeys(SpanAttributes), 'maple.product_event.include')\n OR has(\n arrayMap(\n key -> trimBoth(key),\n splitByChar(',', SpanAttributes['maple.product_event.include'])\n ),\n k\n )\n ),\n SpanAttributes\n ),\n 'Map(String, String)'\n ),\n mapApply(\n (k, v) -> (substring(k, 26), v),\n mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes)\n )\n ) AS Attributes,\n TraceId,\n SpanId\n FROM traces\n WHERE SpanAttributes['maple.product_event.name'] != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '',\n if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index 557c32224..d5208e00f 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "27390bcb1160c0e9f9c1f91aba7c955e5b52a56dda7a7c0bfc6182ef07eaa945" as const +export const projectRevision = "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb" as const export const datasources = [ { @@ -77,7 +77,7 @@ export const datasources = [ { name: "product_events", content: - "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map()\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4", + "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map(),\n TraceId String `json:$.trace_id` DEFAULT '',\n SpanId String `json:$.span_id` DEFAULT ''\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", }, { name: "service_address_resolutions_hourly", @@ -265,7 +265,12 @@ export const pipes = [ { name: "product_events_mv", content: - "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes,\n '' AS TraceId,\n '' AS SpanId\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + }, + { + name: "product_events_traces_mv", + content: + "DESCRIPTION >\n Populates product_events from spans carrying the maple.product_event.name attribute, projecting the span's identity, prop.* attributes, service and TraceId/SpanId so the event links back to the trace that produced it.\n\nNODE product_events_traces_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'trace' AS Source,\n SpanAttributes['session.id'] AS SessionId,\n 0 AS Seq,\n SpanAttributes['maple.product_event.visitor_id'] AS VisitorId,\n SpanAttributes['maple.product_event.user_id'] AS UserId,\n SpanAttributes['maple.product_event.group_id'] AS GroupId,\n 'custom' AS Kind,\n SpanAttributes['maple.product_event.name'] AS EventName,\n domain(SpanAttributes['maple.product_event.url']) AS Host,\n path(SpanAttributes['maple.product_event.url']) AS PagePath,\n SpanAttributes['maple.product_event.url'] AS Url,\n ServiceName,\n mapUpdate(\n CAST(\n mapFilter(\n (k, v) -> NOT startsWith(k, 'maple.product_event.')\n AND (\n NOT has(mapKeys(SpanAttributes), 'maple.product_event.include')\n OR has(\n arrayMap(\n key -> trimBoth(key),\n splitByChar(',', SpanAttributes['maple.product_event.include'])\n ),\n k\n )\n ),\n SpanAttributes\n ),\n 'Map(String, String)'\n ),\n mapApply(\n (k, v) -> (substring(k, 26), v),\n mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes)\n )\n ) AS Attributes,\n TraceId,\n SpanId\n FROM traces\n WHERE SpanAttributes['maple.product_event.name'] != ''\n\nTYPE MATERIALIZED\nDATASOURCE product_events", }, { name: "service_external_edges_hourly_mv", diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 5f30d3c7e..8dbeea254 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -1742,6 +1742,70 @@ export class ProductEventNamesResponse extends Schema.Class( + "ProductEventsForTraceRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + traceId: TraceId, + /** Default 50. */ + limit: Schema.optional(Schema.Number), +}) {} + +export class ProductEventsForTraceResponse extends Schema.Class( + "ProductEventsForTraceResponse", +)({ + data: Schema.Array( + Schema.Struct({ + timestamp: Schema.String, + eventName: Schema.String, + /** The annotated span within the trace. */ + spanId: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + groupId: Schema.String, + visitorId: Schema.String, + sessionId: Schema.String, + /** The event's `maple.product_event.prop.*` attributes, prefix stripped. */ + attributes: Schema.Record(Schema.String, Schema.String), + }), + ), +}) {} + +/** Recent traces behind one event name — the analytics side of the same link. */ +export class ProductEventTraceSamplesRequest extends Schema.Class( + "ProductEventTraceSamplesRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + eventName: Schema.String, + /** Default 20. */ + limit: Schema.optional(Schema.Number), +}) {} + +export class ProductEventTraceSamplesResponse extends Schema.Class( + "ProductEventTraceSamplesResponse", +)({ + data: Schema.Array( + Schema.Struct({ + traceId: Schema.String, + spanId: Schema.String, + timestamp: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + visitorId: Schema.String, + }), + ), +}) {} + export class PodFacetsRequest extends Schema.Class("PodFacetsRequest")({ startTime: TinybirdDateTime, endTime: TinybirdDateTime, @@ -2699,6 +2763,20 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("productEventsForTrace", "/product-events-for-trace", { + payload: ProductEventsForTraceRequest, + success: ProductEventsForTraceResponse, + error: queryEngineEndpointErrors, + }), + ) + .add( + HttpApiEndpoint.post("productEventTraceSamples", "/product-event-trace-samples", { + payload: ProductEventTraceSamplesRequest, + success: ProductEventTraceSamplesResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("executeRawSql", "/execute-raw-sql", { payload: RawSqlExecuteRequest, diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index b05dbfafe..6e98a0339 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2202,6 +2202,20 @@ export const productEvents = defineDatasource("product_events", { Attributes: column(t.map(t.string(), t.string()).defaultExpr("map()"), { jsonPath: "$.attributes", }), + /** + * The trace this event was derived from — set on `Source = 'trace'` rows, + * `''` on every other source. This column IS the link, in both directions: + * the trace view lists the product events a trace produced, and a funnel + * row hands back the exact trace that performed the step. + * + * A real column rather than an `Attributes` key because both directions + * filter on it, and a `Map` lookup on this table reads the whole map per + * row — the cost `product_events` was split out of `session_events` to + * avoid. Last in the schema because `ALTER TABLE … ADD COLUMN` appends. + */ + TraceId: column(t.string().default(""), { jsonPath: "$.trace_id" }), + /** The annotated span within {@link TraceId}. `''` on non-trace rows. */ + SpanId: column(t.string().default(""), { jsonPath: "$.span_id" }), }, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", @@ -2227,6 +2241,17 @@ export const productEvents = defineDatasource("product_events", { type: "bloom_filter", granularity: 4, }, + { + // "Which product events did this trace produce?" — the trace view asks + // it by id with no other predicate, so without this the lookup scans + // every partition in the retained window. Near-unique values, and the + // column is `''` on the overwhelming majority of rows, so a bloom + // filter both prunes hard and stays cheap. + name: "idx_trace_id", + expr: "TraceId", + type: "bloom_filter", + granularity: 4, + }, ], }) diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 59470c592..09f468fb7 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -45,6 +45,10 @@ import { DB_STATEMENT_SQL, DB_SYSTEM_ATTR_SQL, } from "./db-query-shape-sql" +import { + PRODUCT_EVENTS_TRACE_FILTER, + PRODUCT_EVENTS_TRACE_PROJECTION_SQL, +} from "./product-event-attributes" import { DEPLOYMENT_ENV_SQL, MESSAGING_DESTINATION_SQL } from "./semconv-renames" import { NORMALIZED_SPAN_NAME_SQL } from "./span-display-name" @@ -1584,7 +1588,9 @@ export const productEventsMv = defineMaterializedView("product_events_mv", { path(Url) AS PagePath, Url, '' AS ServiceName, - Attributes + Attributes, + '' AS TraceId, + '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom') `, @@ -1592,6 +1598,41 @@ export const productEventsMv = defineMaterializedView("product_events_mv", { ], }) +/** + * Populates `product_events` from spans the customer annotated in their own + * code — `maple.product_event.name` on any span they already emit. + * + * This is the third feed into the table (browser via the MV above, server and + * mobile via `POST /v1/events`), and the only one that carries `TraceId`, which + * is what makes a product event and the trace that produced it navigable from + * either side. + * + * The predicate is one map lookup per incoming span, on the same block every + * other `traces` MV already fires on. It buys no skip-index help — an MV sees + * the insert block, not the table — so it is a real per-span cost at ingest, and + * a deliberately small one: a single `Map` value read on a map the block has + * just materialized anyway. + * + * Column order must match the `product_events` SCHEMA order — enforced by + * `materialized-projection-order.test.ts`. + */ +export const productEventsTracesMv = defineMaterializedView("product_events_traces_mv", { + description: + "Populates product_events from spans carrying the maple.product_event.name attribute, projecting the span's identity, prop.* attributes, service and TraceId/SpanId so the event links back to the trace that produced it.", + datasource: productEvents, + nodes: [ + node({ + name: "product_events_traces_mv_node", + sql: ` + SELECT + ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} + FROM traces + WHERE ${PRODUCT_EVENTS_TRACE_FILTER} + `, + }), + ], +}) + /** * Populates `identity_links` from `session_replays` rows that carry both a * visitor and a user id. diff --git a/packages/domain/src/tinybird/product-event-attributes.test.ts b/packages/domain/src/tinybird/product-event-attributes.test.ts new file mode 100644 index 000000000..a9f89a42a --- /dev/null +++ b/packages/domain/src/tinybird/product-event-attributes.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest" +import { + PRODUCT_EVENTS_TRACE_FILTER, + PRODUCT_EVENTS_TRACE_PROJECTION_SQL, + PRODUCT_EVENT_ATTRIBUTE_NAMESPACE, + PRODUCT_EVENT_INCLUDE_KEY, + PRODUCT_EVENT_NAME_KEY, + PRODUCT_EVENT_PROP_PREFIX, + PRODUCT_EVENT_SOURCE_TRACE, +} from "./product-event-attributes" +import { productEvents } from "./datasources" + +describe("product event span attributes", () => { + it("namespaces every key under maple.", () => { + // The `maple.*` vendor namespace is the convention every custom attribute + // in the repo follows, and it is what keeps these from colliding with an + // OTel semconv key that might later mean something else. + expect(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE.startsWith("maple.")).toBe(true) + expect(PRODUCT_EVENT_NAME_KEY.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + expect(PRODUCT_EVENT_INCLUDE_KEY.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + expect(PRODUCT_EVENT_PROP_PREFIX.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + }) + + it("casts the base map to the target key type", () => { + // Not cosmetic: `SpanAttributes` is keyed `LowCardinality(String)` and + // `product_events.Attributes` is keyed plain `String`. Without the CAST the + // MV's SELECT has a different type from the column it writes, and + // `mapUpdate` has two differently-keyed maps to merge. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain("'Map(String, String)'") + }) + + it("switches the allow-list on key PRESENCE, not on a non-empty value", () => { + // The whole overwrite idiom rests on this. `include: ''` has to mean "no + // span attributes", so the test must be `mapKeys` containment — a `!= ''` + // here would silently turn the documented overwrite into the default + // copy-everything, which is the exact opposite of what the caller asked for. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `has(mapKeys(SpanAttributes), '${PRODUCT_EVENT_INCLUDE_KEY}')`, + ) + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).not.toContain( + `SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}'] != ''`, + ) + }) + + it("trims the allow-list entries", () => { + // `"plan, seats"` is what a human writes. Without the trim the second key + // never matches and the prop silently vanishes from every event. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain("trimBoth") + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `splitByChar(',', SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}'])`, + ) + }) + + it("strips exactly the prop prefix and nothing more", () => { + // ClickHouse `substring` is 1-indexed, so the offset is length + 1. Off by + // one in either direction is silent: one short leaves a leading `.` on + // every prop key, one long eats the first character of every prop NAME — + // `plan` becomes `lan` — and both produce a valid map no breakdown can + // group on. + const offset = /substring\(k, (\d+)\)/.exec(PRODUCT_EVENTS_TRACE_PROJECTION_SQL)?.[1] + expect(offset).toBe(String(PRODUCT_EVENT_PROP_PREFIX.length + 1)) + }) + + it("merges props over the base so an explicit prop wins a collision", () => { + // Argument order in `mapUpdate(base, props)` IS the override rule. Swapped, + // a team overriding a derived value would find their override discarded + // exactly when the key they wanted to correct was already present. + const merge = /mapUpdate\(\s*CAST\(/.exec(PRODUCT_EVENTS_TRACE_PROJECTION_SQL) + expect(merge, "props must be the SECOND mapUpdate argument").not.toBeNull() + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL.indexOf("mapApply")).toBeGreaterThan( + PRODUCT_EVENTS_TRACE_PROJECTION_SQL.indexOf("mapUpdate"), + ) + }) + + it("keeps the control namespace out of the copied attributes", () => { + // Once `prop.*` exists, leaving the namespace in means every explicit prop + // appears twice — as `plan` from the merge and as + // `maple.product_event.prop.plan` from the base — and name/user_id + // duplicate columns this same SELECT already promotes. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `NOT startsWith(k, '${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}')`, + ) + }) + + it("filters on a non-empty name rather than key presence", () => { + // `mapContains` would admit a span whose attribute is set to '' and mint a + // nameless event — a row no funnel can step on and no reader can attribute. + expect(PRODUCT_EVENTS_TRACE_FILTER).toBe(`SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] != ''`) + }) + + it("projects the product_events columns in schema order", () => { + // The MV body is checked structurally by materialized-projection-order, + // which reads the generated manifest. This asserts the same thing about the + // shared constant itself, so a bad edit fails before the manifest is + // regenerated rather than after. + // + // Split on TOP-LEVEL commas only. A line-wise scan was enough while every + // projected column was one line; `Attributes` is now a nested expression + // whose inner `mapFilter`/`arrayMap` lambdas contain both commas and bare + // identifiers, and a naive scan reads `k` and `SpanAttributes` as columns. + const topLevelParts = (input: string): ReadonlyArray => { + const parts: string[] = [] + let start = 0 + let depth = 0 + let inString = false + for (let index = 0; index < input.length; index++) { + const char = input[index] + if (char === "'" && input[index - 1] !== "\\") { + inString = !inString + continue + } + if (inString) continue + if (char === "(") depth++ + if (char === ")") depth-- + if (char === "," && depth === 0) { + parts.push(input.slice(start, index)) + start = index + 1 + } + } + parts.push(input.slice(start)) + return parts.map((part) => part.trim()).filter((part) => part.length > 0) + } + + const projected = topLevelParts(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).map((part) => { + const aliased = /\bAS ([A-Za-z][A-Za-z0-9_]*)$/.exec(part) + return aliased ? aliased[1]! : part + }) + expect(projected).toEqual(Object.keys(productEvents._schema)) + }) + + it("carries a Source distinct from the other three feeds", () => { + // `Source` is the provenance column every reader branches on to tell an + // annotated span from a browser page view or a POST /v1/events row. + expect(PRODUCT_EVENT_SOURCE_TRACE).toBe("trace") + expect(["browser", "server", "mobile"]).not.toContain(PRODUCT_EVENT_SOURCE_TRACE) + }) +}) diff --git a/packages/domain/src/tinybird/product-event-attributes.ts b/packages/domain/src/tinybird/product-event-attributes.ts new file mode 100644 index 000000000..4e8e1fb1a --- /dev/null +++ b/packages/domain/src/tinybird/product-event-attributes.ts @@ -0,0 +1,191 @@ +/** + * The span-attribute contract that turns an instrumented span into a product + * event — "annotate in code, not in the UI". + * + * A customer marks a span they already emit: + * + * ```ts + * span.setAttributes({ + * "maple.product_event.name": "checkout_completed", + * "maple.product_event.user_id": user.id, + * }) + * ``` + * + * The span's own attributes come along whole by default — every key it already + * carries becomes an event property, with no opt-in list and no prefix to + * remember. The span IS the event, so anything worth putting on the span is + * worth breaking the funnel down by. + * + * Two optional controls narrow or replace that default, both themselves span + * attributes, because a materialized view is static SQL per cluster and has no + * per-org config to read: + * + * ```ts + * "maple.product_event.include": "plan,seats" // ONLY these span keys + * "maple.product_event.prop.plan": "pro" // explicit prop, wins ties + * "maple.product_event.include": "" // and together: full overwrite + * ``` + * + * Three tiers out of one mechanism rather than three modes to choose between: + * `include` narrows the base map, `prop.*` merges over it, and an EMPTY + * `include` narrows the base to nothing so the props are all that survive. There + * is no separate "replace" flag to get wrong. + * + * and `product_events_traces_mv` projects it into `product_events` with + * `Source = 'trace'` and the span's `TraceId`/`SpanId` carried through, so the + * event steps in a funnel like any `track()` call AND links back to the trace it + * came from. Nothing is written by hand and nothing is stored twice: the span is + * the record, the product event is its projection. + * + * Why an attribute rather than a UI action: a product event has to be emitted by + * the code path that actually performed the thing, at the moment it performed + * it. A human clicking "this trace was a signup" a day later marks ONE sampled + * trace, cannot be replayed over history, and puts a mutable, user-authored row + * into an append-only fact table. An attribute marks every trace the path + * produces, forever, and is reviewable in the customer's own diff. + * + * These keys are read in exactly two places, which must agree byte for byte: + * `productEventsTracesMv` in `materializations.ts` (managed orgs) and the frozen + * MV body in ClickHouse migration 0024 (BYO clusters). + */ + +/** Vendor namespace prefix. Every key below starts with it. */ +export const PRODUCT_EVENT_ATTRIBUTE_NAMESPACE = "maple.product_event." + +/** + * Required. Its presence — a non-empty value — is the whole predicate: a span + * carrying it becomes a product event, a span without it is ignored. The value + * becomes `EventName`, i.e. the funnel step key. + */ +export const PRODUCT_EVENT_NAME_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}name` + +/** Optional identity. Absent keys project to `''`, which means "unidentified". */ +export const PRODUCT_EVENT_USER_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}user_id` +export const PRODUCT_EVENT_GROUP_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}group_id` +export const PRODUCT_EVENT_VISITOR_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}visitor_id` + +/** + * Optional page context, for events that belong to a URL (a server-rendered + * checkout, a webhook that knows the page it came from). + */ +export const PRODUCT_EVENT_URL_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}url` + +/** + * Optional allow-list: a comma-separated list of span attribute keys, and only + * those are copied into `Attributes`. Whitespace around each entry is trimmed, + * so `"plan, seats"` and `"plan,seats"` mean the same thing. + * + * PRESENCE is what switches the behaviour, not the value — which is why the + * projection tests it with `mapContains` and not `!= ''`. Present and empty + * therefore means "no span attributes at all", and that is the documented way to + * OVERWRITE rather than narrow: set it empty and supply {@link + * PRODUCT_EVENT_PROP_PREFIX} props, and the event carries exactly those. + */ +export const PRODUCT_EVENT_INCLUDE_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}include` + +/** + * Prefix for explicit props. `maple.product_event.prop.plan` lands in + * `Attributes` as `plan`, MERGED OVER whatever the base map produced and winning + * on a key collision — so a team can override one derived value without having + * to enumerate everything else with {@link PRODUCT_EVENT_INCLUDE_KEY}. + */ +export const PRODUCT_EVENT_PROP_PREFIX = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}prop.` + +/** + * OTel's own session key, read as a fallback so a browser-originated trace can + * stitch to the same session its `session_events` rows carry. + */ +export const PRODUCT_EVENT_SESSION_ID_KEY = "session.id" + +/** + * `Source` value for a trace-derived row. Joins `browser` (session_events MV), + * `server` and `mobile` (`POST /v1/events`). + */ +export const PRODUCT_EVENT_SOURCE_TRACE = "trace" + +/** + * The `product_events` projection of one annotated span, shared byte-for-byte + * between the managed materialized view and the BYO-ClickHouse migration's + * frozen copy — as with 0021's browser projection, two copies of this SELECT is + * two chances for the same span to be counted two different ways. + * + * Column order matches the `product_events` SCHEMA order, `TraceId`/`SpanId` + * last because that is where `ALTER TABLE … ADD COLUMN` puts them. + * + * `Kind = 'custom'` deliberately: `Kind` is the page-view predicate every reader + * already branches on, and a trace-derived event is a tracked event, not a page + * view. Provenance lives in `Source`, which is what the trace link reads. + * + * `Seq = 0` like every other direct row; ordering inside a millisecond comes + * from the span's own nanosecond `Timestamp`. + * + * `Attributes` is built in two halves and merged, `mapUpdate` letting the props + * win a collision: + * + * base = span attributes, minus the `maple.product_event.*` control + * namespace, optionally narrowed to `include`'s allow-list + * props = `maple.product_event.prop.*`, prefix stripped + * + * The default — neither control set — is the whole span map, which is the + * expensive choice and the deliberate one: a server span carries its full + * HTTP/DB semconv surface, and `product_events` keeps 365 days against raw + * `traces`' 30, so an annotated span's attributes outlive the span by a factor + * of twelve. What it buys is that nothing has to be declared to get started, and + * `include` is the lever for a team that has measured the cost and wants it back. + * + * The control namespace IS stripped from the base, unlike an earlier cut of + * this projection. Once `prop.*` exists, leaving it in means every explicit prop + * appears twice — once as `plan` from the merge and once as + * `maple.product_event.prop.plan` from the base — and `name`/`user_id` duplicate + * columns this same SELECT already promotes. + * + * The whole expression only evaluates for rows that pass the WHERE below, i.e. + * annotated spans, so its cost is paid per product event rather than per span. + * The predicate itself stays a single map lookup. + */ +export const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + '${PRODUCT_EVENT_SOURCE_TRACE}' AS Source, + SpanAttributes['${PRODUCT_EVENT_SESSION_ID_KEY}'] AS SessionId, + 0 AS Seq, + SpanAttributes['${PRODUCT_EVENT_VISITOR_ID_KEY}'] AS VisitorId, + SpanAttributes['${PRODUCT_EVENT_USER_ID_KEY}'] AS UserId, + SpanAttributes['${PRODUCT_EVENT_GROUP_ID_KEY}'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] AS EventName, + domain(SpanAttributes['${PRODUCT_EVENT_URL_KEY}']) AS Host, + path(SpanAttributes['${PRODUCT_EVENT_URL_KEY}']) AS PagePath, + SpanAttributes['${PRODUCT_EVENT_URL_KEY}'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, '${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}') + AND ( + NOT has(mapKeys(SpanAttributes), '${PRODUCT_EVENT_INCLUDE_KEY}') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, ${PRODUCT_EVENT_PROP_PREFIX.length + 1}), v), + mapFilter((k, v) -> startsWith(k, '${PRODUCT_EVENT_PROP_PREFIX}'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +/** + * The predicate, also shared. `!= ''` rather than `mapContains`: a key present + * with an empty value would otherwise mint an event with no name, which is a row + * no funnel can step on and no reader can attribute. + */ +export const PRODUCT_EVENTS_TRACE_FILTER = `SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] != ''` diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index c07c7bb26..d8c4ed8e8 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -1103,6 +1103,26 @@ SELECT LIMIT 100 FORMAT JSON +-- builder:product-events:productEventsForTraceQuery:default [d151e174] +SELECT + Timestamp AS timestamp, + EventName AS eventName, + SpanId AS spanId, + ServiceName AS serviceName, + UserId AS userId, + GroupId AS groupId, + VisitorId AS visitorId, + SessionId AS sessionId, + Attributes AS attributes + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND TraceId = '4bf92f3577b34da6a3ce929d0e0e4736' + ORDER BY timestamp ASC, spanId ASC + LIMIT 50 + FORMAT JSON + -- builder:product-events:productEventsFunnelBreakdownQuery:attribute-session-step [ac39fa69] SELECT group AS group, @@ -1467,6 +1487,24 @@ SELECT ORDER BY step ASC FORMAT JSON +-- builder:product-events:productEventTraceSamplesQuery:default [ee1608d5] +SELECT + TraceId AS traceId, + SpanId AS spanId, + Timestamp AS timestamp, + ServiceName AS serviceName, + UserId AS userId, + VisitorId AS visitorId + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND EventName = 'checkout_completed' + AND TraceId != '' + ORDER BY timestamp DESC + LIMIT 20 + FORMAT JSON + -- builder:service-endpoints:serviceEndpointsSummaryQuery:default [3e379104] SELECT bSpanName AS spanName, diff --git a/packages/query-engine/src/ch/builder-fixtures.ts b/packages/query-engine/src/ch/builder-fixtures.ts index 69d8f7adf..7d76ebf59 100644 --- a/packages/query-engine/src/ch/builder-fixtures.ts +++ b/packages/query-engine/src/ch/builder-fixtures.ts @@ -311,6 +311,29 @@ const productEventsFixtures: ReadonlyArray = [ window, ), }, + // The two directions of the trace ↔ product-event link. Both are + // single-predicate lookups on `TraceId`, so what the sweep is watching for is + // that neither grows a `Map` read or loses its `OrgId`/time bounds. + { + module: "product-events", + name: "productEventsForTraceQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.productEventsForTraceQuery({ limit: 50 }), { + ...window, + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }), + }, + { + module: "product-events", + name: "productEventTraceSamplesQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.productEventTraceSamplesQuery({ limit: 20 }), { + ...window, + eventName: "checkout_completed", + }), + }, ] export const builderFixtures: ReadonlyArray = [ diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 04b1e4433..c9f8d05c2 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -189,6 +189,10 @@ export { productEventsFunnelBreakdownRowSchema, productEventNamesQuery, productEventNamesRowSchema, + productEventsForTraceQuery, + productEventForTraceRowSchema, + productEventTraceSamplesQuery, + productEventTraceSampleRowSchema, ProductEventsFunnelError, FUNNEL_MAX_STEPS, FUNNEL_BREAKDOWN_MAX_GROUPS, @@ -196,6 +200,10 @@ export { type FunnelKeyBy, type FunnelSessionDimension, type FunnelBreakdownBy, + type ProductEventsForTraceOpts, + type ProductEventForTraceOutput, + type ProductEventTraceSamplesOpts, + type ProductEventTraceSampleOutput, type ProductEventsFunnelOpts, type ProductEventsFunnelOutput, type ProductEventsFunnelBreakdownOpts, diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index eceedfcea..d3490338b 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -694,3 +694,127 @@ export function productEventNamesQuery( .limit(limit) .format("JSON") } + +// Trace ↔ product event linking +// +// A span the customer annotated with `maple.product_event.name` lands in +// `product_events` as a `Source = 'trace'` row carrying its `TraceId`/`SpanId` +// (ClickHouse migration 0024 / `product_events_traces_mv`). That column is the +// link, and these two queries are the two directions of walking it: +// +// trace → its product events `productEventsForTraceQuery` +// event → the traces behind it `productEventTraceSamplesQuery` +// +// Both filter `TraceId` directly rather than reaching through `Attributes`, +// because both are single-predicate lookups that must prune on the table's +// `idx_trace_id` bloom filter — a `Map` read would take the whole map per row on +// the one table that exists to avoid exactly that. + +export const productEventForTraceRowSchema = Schema.Struct({ + timestamp: Schema.String, + eventName: Schema.String, + spanId: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + groupId: Schema.String, + visitorId: Schema.String, + sessionId: Schema.String, + attributes: Schema.Record(Schema.String, Schema.String), +}) +export type ProductEventForTraceOutput = typeof productEventForTraceRowSchema.Type + +export interface ProductEventsForTraceOpts { + /** Default 50 — a single trace producing more than this is pathological. */ + readonly limit?: number +} + +/** + * The product events one trace produced, oldest first — the trace view's + * "this request performed these things" panel. + * + * Deliberately NOT time-bounded beyond the caller's window: a trace id is + * already near-unique, and `idx_trace_id` prunes on it, so adding a narrow time + * predicate would only risk missing an event whose span started either side of + * the window the caller happened to pass. Callers still bound the range, because + * `OrgId`/`Timestamp` lead the sorting key and partition pruning is what keeps + * this off every retained day. + * + * `Source` is not filtered: a row carrying a `TraceId` can only have come from + * the trace projection, so filtering it would be a second way to say the same + * thing and a second thing to keep in sync. + */ +export function productEventsForTraceQuery( + opts: ProductEventsForTraceOpts = {}, +): CHQuery { + return from(ProductEvents) + .select(($) => ({ + timestamp: $.Timestamp, + eventName: $.EventName, + spanId: $.SpanId, + serviceName: $.ServiceName, + userId: $.UserId, + groupId: $.GroupId, + visitorId: $.VisitorId, + sessionId: $.SessionId, + attributes: $.Attributes, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + $.TraceId.eq(param.string("traceId")), + ]) + .orderBy(["timestamp", "asc"], ["spanId", "asc"]) + .limit(opts.limit ?? 50) + .format("JSON") +} + +export const productEventTraceSampleRowSchema = Schema.Struct({ + traceId: Schema.String, + spanId: Schema.String, + timestamp: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + visitorId: Schema.String, +}) +export type ProductEventTraceSampleOutput = typeof productEventTraceSampleRowSchema.Type + +export interface ProductEventTraceSamplesOpts { + /** Default 20. */ + readonly limit?: number +} + +/** + * Recent traces behind one event name, newest first — the other direction: + * standing on a funnel step or an event in the analytics list, go look at what + * actually happened. + * + * Only annotated rows can answer this, so `TraceId != ''` is the filter rather + * than `Source = 'trace'`: it is the same set, and it is the predicate that + * makes the result USEFUL (a row with no trace id is not a sample of anything), + * so stating it that way keeps the query honest about why it excludes the + * browser and `/v1/events` rows. + */ +export function productEventTraceSamplesQuery( + opts: ProductEventTraceSamplesOpts = {}, +): CHQuery { + return from(ProductEvents) + .select(($) => ({ + traceId: $.TraceId, + spanId: $.SpanId, + timestamp: $.Timestamp, + serviceName: $.ServiceName, + userId: $.UserId, + visitorId: $.VisitorId, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + $.EventName.eq(param.string("eventName")), + $.TraceId.neq(""), + ]) + .orderBy(["timestamp", "desc"]) + .limit(opts.limit ?? 20) + .format("JSON") +} diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index fb63eeece..8265b1c56 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -746,6 +746,13 @@ export const ProductEvents = table("product_events", { ServiceName: T.string, // track() props. Attributes: T.map(T.string, T.string), + // The trace this event was derived from — non-empty only on Source='trace' + // rows, i.e. spans the customer annotated with `maple.product_event.name`. + // The link in both directions: trace view → its product events, funnel row → + // the trace that performed the step. + TraceId: T.string, + // The annotated span within TraceId. '' on every other source. + SpanId: T.string, }) // (VisitorId, UserId) pairs observed together on a session_replays row. diff --git a/packages/query-engine/src/registry/product-events.ts b/packages/query-engine/src/registry/product-events.ts index c11545d34..d11fb59f3 100644 --- a/packages/query-engine/src/registry/product-events.ts +++ b/packages/query-engine/src/registry/product-events.ts @@ -1,7 +1,9 @@ import type { ProductEventNamesRequest, + ProductEventsForTraceRequest, ProductEventsFunnelBreakdownRequest, ProductEventsFunnelRequest, + ProductEventTraceSamplesRequest, } from "@maple/domain/http" import * as CH from "../ch" import { timeRangeCache } from "../runtime/query-engine" @@ -81,3 +83,41 @@ export const productEventNames = defineQuery({ { orgId, startTime: payload.startTime, endTime: payload.endTime }, ), }) + +// The trace ↔ product-event link, both directions. `list` rather than +// `aggregation`: each reads a handful of rows off a single equality predicate, +// and paying the aggregation profile's settings for a bloom-filter point lookup +// is the wrong trade. +// +// A flat 60s rather than `timeRangeCache`, whose TTL and key-snap widen with the +// range: both of these are point lookups whose answer does not change with how +// much time the caller asked about, and a completed trace's events never change +// at all. 60s covers the burst of re-reads a panel does while someone clicks +// around one trace, and is short enough that an event annotated a minute ago +// shows up in the samples list. + +export const productEventsForTrace = defineQuery({ + id: "productEventsForTrace", + profile: "list", + cache: 60, + compile: (payload: ProductEventsForTraceRequest, orgId: string) => + CH.compile(CH.productEventsForTraceQuery({ limit: payload.limit ?? 50 }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + traceId: payload.traceId, + }), +}) + +export const productEventTraceSamples = defineQuery({ + id: "productEventTraceSamples", + profile: "list", + cache: 60, + compile: (payload: ProductEventTraceSamplesRequest, orgId: string) => + CH.compile(CH.productEventTraceSamplesQuery({ limit: payload.limit ?? 20 }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + eventName: payload.eventName, + }), +}) diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index a424554a9..8484f0884 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -56,7 +56,13 @@ import { makeTimeRangeCachePolicy, timeRangeCache } from "../runtime/query-engin import { defineQuery } from "./query-definition" export { logsCount, logsTimeseries } from "./logs" -export { productEventsFunnel, productEventsFunnelBreakdown, productEventNames } from "./product-events" +export { + productEventsFunnel, + productEventsFunnelBreakdown, + productEventNames, + productEventsForTrace, + productEventTraceSamples, +} from "./product-events" /** * Declarative compile, execution, and cache policy. Handlers retain response From c51c20baaa082dcb3a6eecb7053508bf8d9afb2c Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 1 Sep 2026 13:12:30 +0200 Subject: [PATCH 2/5] fix(product-events): close two data-loss paths found in review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-review with four adversarial passes (warehouse SQL, migration safety, API boundary, UI) turned up two bugs that would have shipped. 1. BYO ingest would have dropped every /v1/events batch for unmigrated orgs. Widening the product_events datasource regenerated the Rust gateway's insert mapping to name TraceId/SpanId, but the readiness gate stayed at schema version 21 because 0024 is requiredForIngest: false. A BYO org stamped 21-23 is therefore still routed to its own cluster, where the INSERT fails on the unknown column, retries, trips the breaker and drops the batch. Reproduced against a pre-0024 table: Code 16, NO_SUCH_COLUMN_IN_TABLE. Fixed at the declaration rather than the flag: TraceId/SpanId now carry no jsonPath, so generate-clickhouse-insert-mappings skips them and the gateway never names them — the same shape service_usage's MV-only columns already use. requiredForIngest: false is honest again, and no BYO org is un-readied over columns none of their writers touch. The migration comment now says which fact it depends on, since that fact lives in another file. 2. Dropping product_events_mv across the whole trace backfill was a permanent hole in page views. The ordering was copied from 0021, where the bracket was forced — there the backfill WAS the browser feed. Here the backfill reads traces and the view reads session_events, so the bracket bought nothing while the chunked backfill ran for up to 400 workflow steps, and every navigation row ingested meanwhile was never projected. The view is now recreated immediately after its drop; the outage is one statement wide. Also from the review: - The idempotency DELETE is now scoped to the backfill's own source window (Timestamp >= (SELECT min(Timestamp) FROM traces)) in both 0024 and the local edge. Unbounded, a late re-apply cleared 365 days of trace rows and rebuilt only the 30 that traces still holds. Verified: 200 rows -> 170 kept, browser rows untouched. - limit is constrained at the HTTP boundary (RowLimit: int, 1..1000). The builder inlines a limit into the SQL text, so limit: -1 and limit: 1e21 were 500s rather than 400s and limit: 1e9 an unbounded scan — the bucket_seconds mistake in a second costume, one field over from a comment citing that rule. - Deleted the two exported row schemas: both were byte-identical to what the builder derives and neither was passed to compile, so they were a contract nothing enforced. Declared schemas earn their place by narrowing. - Removed toStringRecord. Rows are decoded before reaching it, so the driver quirk it defended against cannot occur. - The trace panel no longer mounts an atom with empty time bounds before its own guard runs — that manufactured a swallowed decode error and exported a failure span per render, and only missed the network because TinybirdDateTime rejects "". Window resolution now happens before the child that queries. - Row keys include the index: SpanId is '' on any row that reached the table without a span, so two same-named events in one trace collided. - A row with no span to select is a plain row, not a disabled button, which had removed its whole content from the tab order with no visual cue. - The analytics panel renders an error state instead of silence. It mounts because the user asked for it and empty is a meaningful answer there, so swallowing a failure answered their question wrongly. - Docs: the falsified requiredForIngest claim, plus a warning that the managed populate is one-shot and overlap-prone (no DELETE step exists on Tinybird, so BYO risks a gap where managed risks duplicates). Unchanged and verified clean by the review: the Attributes expression (merge direction, substring offset, all three include tiers, key types, lambda capture), column order, OrgId scoping and time bounds on both queries, and cache keys — the identity embeds the full payload and is prefixed with orgId. --- .../src/routes/internal/query-engine.http.ts | 21 +---- .../v13-to-v14-product-events-from-traces.ts | 15 ++- apps/cli/src/server/schema/local-inserts.json | 2 +- apps/ingest/src/clickhouse_insert_mappings.rs | 8 +- .../analytics/product-event-trace-samples.tsx | 29 +++++- .../traces/trace-product-events.tsx | 92 +++++++++++++------ docs/product-events-funnels.md | 23 ++++- .../0024_product_events_from_traces.ts | 59 +++++++++--- .../domain/src/generated/clickhouse-schema.ts | 2 +- .../generated/tinybird-project-manifest.ts | 4 +- packages/domain/src/http/query-engine.ts | 41 +++++++-- packages/domain/src/tinybird/datasources.ts | 14 ++- packages/query-engine/src/ch/index.ts | 2 - .../src/ch/queries/product-events.ts | 49 +++++----- 14 files changed, 257 insertions(+), 104 deletions(-) diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 7541f05be..d749b0051 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -213,17 +213,6 @@ const coerceStatusCode = (value: string): StatusCode => // point at fresh traces). Probing the last 48h first prunes to ~2 daily // partitions; only older traces fall back to the unbounded every-partition // probe. -/** - * A ClickHouse `Map(String, String)` as the wire hands it over. Non-string - * values are stringified rather than dropped: the column's own type guarantees - * strings, so anything else means a driver quirk, and losing the key silently - * would be worse than showing its coerced value. - */ -const toStringRecord = (value: unknown): Record => { - if (typeof value !== "object" || value === null || Array.isArray(value)) return {} - return Object.fromEntries(Object.entries(value).map(([key, entry]) => [key, String(entry)])) -} - const PROBE_RECENT_WINDOW_MS = 48 * 3_600_000 const toServicePlatformRow = (row: CH.ServicePlatformsOutput) => { @@ -2079,10 +2068,7 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) - // Both directions of the trace ↔ product-event link. `Attributes` is - // the one field that is not a scalar: the warehouse hands back a - // Map as an object, and a row that somehow arrives without one - // degrades to `{}` rather than failing the whole panel. + // Both directions of the trace ↔ product-event link. .handle("productEventsForTrace", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context @@ -2097,7 +2083,10 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query groupId: String(row.groupId), visitorId: String(row.visitorId), sessionId: String(row.sessionId), - attributes: toStringRecord(row.attributes), + // Already decoded as Record by the derived row + // schema — a non-string value fails that decode long before it + // reaches here, so there is nothing left to coerce. + attributes: row.attributes, })), }) }), diff --git a/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts b/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts index 3e1eb32b5..5e7857541 100644 --- a/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts +++ b/apps/cli/src/server/local-store-migrations/v13-to-v14-product-events-from-traces.ts @@ -308,7 +308,15 @@ const apply = async (context: MigrationModuleContext): Promise ) return context.openTarget( (db) => { - db.exec("DELETE FROM product_events WHERE Source = 'trace'") + // Scoped to the backfill's own source window, matching migration 0024: + // `traces` keeps 30 days and `product_events` 365, so an unbounded + // delete would destroy trace rows a late re-run can no longer rebuild. + // A v13 store has no trace rows at all — the source did not exist — so + // this is a no-op on the edge it actually runs on, and correct on any + // resume or re-apply that finds some. + db.exec( + "DELETE FROM product_events WHERE Source = 'trace' AND Timestamp >= (SELECT min(Timestamp) FROM traces)", + ) db.exec( `INSERT INTO product_events (${PRODUCT_EVENTS_TRACE_COLUMNS}) SELECT ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, ) @@ -412,7 +420,7 @@ const dispositions: ReadonlyArray = [ classification: "derived", disposition: "preserve-exact", guarantee: - "Two columns are added as metadata-only defaults, no part is rewritten, and the count of rows whose Source is not 'trace' is verified byte-identical after the backfill.", + "Two columns are added as metadata-only defaults, no part is rewritten, and the count of rows whose Source is not 'trace' is verified unchanged after the backfill. Counts, not contents: the byte-level claim rests on ADD COLUMN being metadata-only, which this edge does not independently verify.", }, { // Fully rebuilt within the raw window, then accrued: unlike the last two @@ -424,6 +432,9 @@ const dispositions: ReadonlyArray = [ guarantee: "Every annotated span still inside raw traces retention is re-projected, and the resulting row count is verified to equal the count of matching spans. Annotated spans older than that window are gone from traces and cannot be rebuilt; the table accrues them from the migration forward.", preservationInterval: "the raw traces retention window", + // The schema default. A store running a custom raw-telemetry floor (see + // `readRawTelemetryRetentionDays`) keeps more or less than this, and the + // backfill follows the store rather than this number. sourceRetentionDays: 30, targetRetentionDays: 365, }, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index ec2ae3821..63c5def8d 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb", + "projectRevision": "5b176340a54f37477fae211fa837c6badd494e1a7cf48c9ea923b58c28783fdb", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index d1cf2cfda..f5d678933 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb"; +pub const PROJECT_REVISION: &str = "5b176340a54f37477fae211fa837c6badd494e1a7cf48c9ea923b58c28783fdb"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse @@ -85,9 +85,9 @@ pub const DATASOURCES: &[InsertMapping] = &[ InsertMapping { datasource: "product_events", table: "product_events", - columns: &["OrgId", "Timestamp", "Source", "SessionId", "Seq", "VisitorId", "UserId", "GroupId", "Kind", "EventName", "Host", "PagePath", "Url", "ServiceName", "Attributes", "TraceId", "SpanId"], - selects: &["__ORG__", "timestamp", "source", "session_id", "seq", "visitor_id", "user_id", "group_id", "kind", "event_name", "host", "page_path", "url", "service_name", "attributes", "trace_id", "span_id"], - input_schema: "timestamp DateTime64(9), source LowCardinality(String), session_id String, seq UInt32, visitor_id String, user_id String, group_id String, kind LowCardinality(String), event_name String, host LowCardinality(String), page_path String, url String, service_name LowCardinality(String), attributes Map(String, String), trace_id String, span_id String", + columns: &["OrgId", "Timestamp", "Source", "SessionId", "Seq", "VisitorId", "UserId", "GroupId", "Kind", "EventName", "Host", "PagePath", "Url", "ServiceName", "Attributes"], + selects: &["__ORG__", "timestamp", "source", "session_id", "seq", "visitor_id", "user_id", "group_id", "kind", "event_name", "host", "page_path", "url", "service_name", "attributes"], + input_schema: "timestamp DateTime64(9), source LowCardinality(String), session_id String, seq UInt32, visitor_id String, user_id String, group_id String, kind LowCardinality(String), event_name String, host LowCardinality(String), page_path String, url String, service_name LowCardinality(String), attributes Map(String, String)", }, ]; diff --git a/apps/web/src/components/analytics/product-event-trace-samples.tsx b/apps/web/src/components/analytics/product-event-trace-samples.tsx index f96ec8a66..5f94df68d 100644 --- a/apps/web/src/components/analytics/product-event-trace-samples.tsx +++ b/apps/web/src/components/analytics/product-event-trace-samples.tsx @@ -15,6 +15,12 @@ import { ChartBarTrendUpIcon } from "@/components/icons" * a finding about the event, it just means the event was not emitted from a * span, and saying so on every browser event would be noise on the majority of * them. + * + * A FAILURE is not silent, though, unlike the trace-page panel. This one mounts + * because the user explicitly filtered to an event and asked for it, and empty + * is a meaningful answer here — so swallowing an error would answer their + * question wrongly ("this event has no traces") with no way to tell and no way + * to retry. */ export function ProductEventTraceSamples({ eventName, @@ -34,14 +40,16 @@ export function ProductEventTraceSamples({ .onSuccess((response) => { if (response.data.length === 0) return null return ( -
    +

    Traces behind “{eventName}”

      - {response.data.map((sample) => ( -
    • + {/* Index included: at-least-once ingest can duplicate a row, and + one trace can fire the event from several spans. */} + {response.data.map((sample, index) => ( +
    • {sample.traceId.slice(0, 8)} @@ -72,6 +80,17 @@ export function ProductEventTraceSamples({
    ) }) - .onError(() => null) + .onError(() => ( +
    +
    + +

    Traces behind “{eventName}”

    +
    +

    + Could not load traces for this event. This is a query failure, not an empty result — + reload to try again. +

    +
    + )) .orElse(() => null) } diff --git a/apps/web/src/components/traces/trace-product-events.tsx b/apps/web/src/components/traces/trace-product-events.tsx index 71c14e61d..e2f10b7cf 100644 --- a/apps/web/src/components/traces/trace-product-events.tsx +++ b/apps/web/src/components/traces/trace-product-events.tsx @@ -35,20 +35,39 @@ export function TraceProductEvents({ onSelectSpan: (spanId: string) => void }) { const traceStartMs = parseWarehouseDateTime(traceStartTime) - const window = Number.isNaN(traceStartMs) - ? undefined - : { - startTime: formatWarehouseDateTime(traceStartMs - WINDOW_MARGIN_MS), - endTime: formatWarehouseDateTime(traceStartMs + totalDurationMs + WINDOW_MARGIN_MS), - } - - const result = useAtomValue( - productEventsForTraceResultAtom({ - data: { traceId, startTime: window?.startTime ?? "", endTime: window?.endTime ?? "" }, - }), + if (Number.isNaN(traceStartMs)) return null + return ( + ) +} - if (window === undefined) return null +/** + * Split out so the unparseable-timestamp case never constructs an atom key at + * all. Folding the guard in after the `useAtomValue` — passing `?? ""` to + * satisfy the required time fields — mounts the atom, fails `decodeInput` + * against `TinybirdDateTime`, and exports a `QueryEngine.getProductEventsForTrace` + * failure span on every render, for a query nobody wanted. It happened not to + * reach the network only because that schema rejects `""`; a later change making + * the window optional-with-fallback (which the logs client already does) would + * have turned it into a real full-window warehouse read. + */ +function LoadedTraceProductEvents({ + traceId, + startTime, + endTime, + onSelectSpan, +}: { + traceId: string + startTime: string + endTime: string + onSelectSpan: (spanId: string) => void +}) { + const result = useAtomValue(productEventsForTraceResultAtom({ data: { traceId, startTime, endTime } })) return Result.builder(result) .onSuccess((response) => { @@ -61,9 +80,14 @@ export function TraceProductEvents({ {response.data.length}
      - {response.data.map((event) => ( + {/* Index, not spanId+eventName: `SpanId` is '' on any row that + reached the table without a span, so two same-named events in + one trace collide on that key — and at-least-once ingest can + duplicate a row outright. The list is ordered by the server + and never reordered client-side, so the index is stable. */} + {response.data.map((event, index) => ( @@ -89,24 +113,40 @@ function ProductEventRow({ const person = event.userId || event.groupId || event.visitorId const props = Object.entries(event.attributes) + const content = ( + <> + {event.eventName} + {event.serviceName === "" ? null : ( + {event.serviceName} + )} + {person === "" ? null : · {person}} + {props.map(([key, value]) => ( + + {key}: {value} + + ))} + + ) + const rowClassName = "flex w-full flex-wrap items-center gap-x-2 gap-y-1 px-3 py-2 text-left text-xs" + + // A row with no span to select is a plain row, not a disabled button. As a + // disabled button its whole content — event name, identity, props — leaves + // the tab order and is unreachable by keyboard, while a mouse user gets no + // cue at all: there is no dimming, only a hover highlight that silently + // doesn't appear. The information is worth reading either way; only the + // navigation is unavailable. + if (event.spanId === "") { + return
    • {content}
      • + } + return (
      • ) diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index b661466e8..dc50ed88b 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -391,10 +391,29 @@ lookup. adds the two columns, then an explicit `tb` populate from `traces` (bounded by its 30-day TTL). Blocked on the same manual step the rest of this document's checklist is — see `project_product_events_tinybird_rollout_pending`. -2. **BYO ClickHouse**: migration 0024, `requiredForIngest: false` — the gateway writes neither new - column, so ingest routing is not un-readied. Backfills the trace half from `traces` itself. + + **The populate is one-shot and overlap-prone.** Unlike BYO and local, the managed surface has no + `DELETE WHERE Source = 'trace'` step, so running it twice double-inserts, and running it after + the MV is already live double-counts every annotated span ingested between MV creation and the + populate's own snapshot. BYO risks a gap; managed risks duplicates. Same caveat 0014 and 0021 + accepted — but on a table feeding customer-visible funnels, a double-counted conversion is worse + than a missing one. Populate once, immediately after deploy, and if it fails partway prefer + deleting the trace rows by hand over re-running it blind. +2. **BYO ClickHouse**: migration 0024, `requiredForIngest: false`. That is safe for one reason + worth knowing before anyone touches `datasources.ts`: `TraceId`/`SpanId` are declared with **no + `jsonPath`**, so the insert-mapping generator omits them and the Rust gateway's + `INSERT INTO product_events (…)` never names them — a cluster stamped below 24 still accepts + every row it sends. Give those columns a `jsonPath` and the flag becomes a data-loss bug: the + readiness gate still says 21, so unmigrated BYO orgs keep routing to their own cluster, where + the INSERT fails on the unknown column, retries, trips the breaker and drops the batch. + Backfills the trace half from `traces` itself. 3. **Local CLI**: local schema v13 → v14, same backfill. +Both BYO and local scope their idempotency `DELETE` to `Timestamp >= (SELECT min(Timestamp) FROM +traces)` rather than deleting all trace rows. `product_events` keeps 365 days and `traces` 30, so an +unbounded delete on a *late* re-apply would clear a year of funnel history and rebuild only a month +of it. + ### Not in this cut - **No MCP tool.** `list_product_events` still returns names only, and `inspect_trace` does not diff --git a/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts b/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts index f47cefee0..b92130497 100644 --- a/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts +++ b/packages/domain/src/clickhouse/migrations/0024_product_events_from_traces.ts @@ -127,19 +127,32 @@ export const productEventsTracesBackfill: BackfillSpec = { * rather than a repair, and the re-create is what keeps the view's text and the * table's schema describing the same thing. * - * Re-runnable by construction: `ADD COLUMN IF NOT EXISTS`, views dropped before - * the backfill, and `DELETE WHERE Source = 'trace'` clears exactly and only what - * the backfill is about to re-insert. + * Re-runnable by construction: `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT + * EXISTS`, and a `DELETE` scoped to the backfill's own source window so it + * clears exactly and only what the following backfill re-inserts — at any point + * in the table's life, not just on first apply. * * **BYO ClickHouse only.** Managed orgs get the same view via `tinybird deploy` * from `materializations.ts`, with the populate as an explicit `tb` step at * deploy time (see 0014 and 0021 — the SDK has no populate option). * - * `requiredForIngest: false`. The two new columns are MV- and backfill-written; - * the Rust gateway's `/v1/events` path does not emit them, and a cluster without - * them still accepts every row the gateway sends. Bumping - * `clickHouseSchemaVersion` would un-ready ingest routing for every BYO-CH org - * over a feature none of their existing writers touch. + * `requiredForIngest: false`, and that rests on ONE fact worth stating plainly + * because it is not local to this file: `TraceId`/`SpanId` are declared in + * `datasources.ts` WITHOUT a `jsonPath`, so + * `scripts/generate-clickhouse-insert-mappings.ts` omits them and the Rust + * gateway's `INSERT INTO product_events (…)` never names them. A cluster stamped + * below 24 therefore still accepts every row the gateway sends. + * + * Give those columns a `jsonPath` and this flag becomes a data-loss bug: the + * readiness gate compares `stamped >= clickHouseSchemaVersion`, which stays at + * 21, so a BYO org that has not applied 0024 is still routed to its own cluster + * — where the INSERT fails on the unknown column, retries, trips the breaker and + * drops the batch. The gate's safety argument is "an older binary writing into a + * newer schema", and that would be the inverse. + * + * Bumping `clickHouseSchemaVersion` instead would un-ready ingest routing for + * every BYO-CH org over a feature none of their existing writers touch, which is + * why the column declaration is the right place to solve it. */ export const migration_0024_product_events_from_traces = { version: 24, @@ -150,12 +163,17 @@ export const migration_0024_product_events_from_traces = { "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''", "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''", "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", - "DROP VIEW IF EXISTS product_events_traces_mv", + // The browser view is dropped and IMMEDIATELY recreated, before anything + // slow runs. 0021 bracketed its backfill with the drop because there the + // backfill WAS the browser feed and two writers of the same rows had to be + // impossible. Here the backfill reads `traces` and the browser view reads + // `session_events`, so the bracket buys nothing and costs everything: the + // backfill is chunked across up to 400 durable workflow steps over 30 days + // of spans, and every `session_events` navigation row ingested while the + // view is gone is never projected at all. That is a permanent hole in page + // views, not the "ordering hygiene" an earlier draft of this comment + // called it. Recreated here, the outage is one statement wide. "DROP VIEW IF EXISTS product_events_mv", - // Idempotency for the trace half only — never touches a browser row or a - // directly ingested one. - "DELETE FROM product_events WHERE Source = 'trace'", - productEventsTracesBackfill, // Frozen copy of the browser projection as of 0021, plus the two new // columns explicitly at `''`. Same freezing rule as above: this is what // the view's body was at this point in history. @@ -180,6 +198,21 @@ SELECT '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom')`, + "DROP VIEW IF EXISTS product_events_traces_mv", + // Idempotency for the trace half only — never a browser row, never a + // directly ingested one. + // + // BOUNDED BY THE BACKFILL'S OWN WINDOW, which is what makes the claim + // above true on a LATE re-run rather than only on first apply. `traces` + // keeps 30 days and `product_events` keeps 365, so by day 100 the table + // holds trace rows the backfill can no longer rebuild. An unbounded + // `DELETE WHERE Source = 'trace'` would clear all 100 days and re-insert + // 30 — silently destroying 70 days of a customer's funnel history on a + // re-apply after a lost bookkeeping row. Scoping the delete to + // `min(Timestamp)` of the source means it removes exactly the rows the + // following backfill is about to write, and nothing older. + "DELETE FROM product_events WHERE Source = 'trace' AND Timestamp >= (SELECT min(Timestamp) FROM traces)", + productEventsTracesBackfill, `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS SELECT ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index a001e1164..816a4edfd 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb" as const +export const projectRevision = "5b176340a54f37477fae211fa837c6badd494e1a7cf48c9ea923b58c28783fdb" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS alert_checks (\n OrgId LowCardinality(String),\n RuleId String,\n GroupKey String,\n Timestamp DateTime64(3),\n Status LowCardinality(String),\n SignalType LowCardinality(String),\n Comparator LowCardinality(String),\n Threshold Float64,\n ObservedValue Nullable(Float64),\n SampleCount UInt32,\n WindowMinutes UInt16,\n WindowStart DateTime64(3),\n WindowEnd DateTime64(3),\n ConsecutiveBreaches UInt16,\n ConsecutiveHealthy UInt16,\n IncidentId Nullable(String),\n IncidentTransition LowCardinality(String),\n EvaluationDurationMs UInt32,\n ErrorMessage Nullable(String),\n ErrorCategory LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, RuleId, GroupKey, Timestamp)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index d5208e00f..350525fb6 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb" as const +export const projectRevision = "5b176340a54f37477fae211fa837c6badd494e1a7cf48c9ea923b58c28783fdb" as const export const datasources = [ { @@ -77,7 +77,7 @@ export const datasources = [ { name: "product_events", content: - "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map(),\n TraceId String `json:$.trace_id` DEFAULT '',\n SpanId String `json:$.span_id` DEFAULT ''\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map(),\n TraceId String `json:$.TraceId` DEFAULT '',\n SpanId String `json:$.SpanId` DEFAULT ''\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", }, { name: "service_address_resolutions_hourly", diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 8dbeea254..cf5091067 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -42,6 +42,31 @@ const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0 }), ) +/** + * A `LIMIT` a client is allowed to ask for. + * + * The builder INLINES a limit into the SQL text (`raw(String(Math.round(v)))`) + * rather than binding it, so an unchecked `Schema.Number` here is the + * `bucket_seconds: 1.5` mistake in a second costume: `limit: -1` and + * `limit: 1e21` (which stringifies as `1e+21`) both reach ClickHouse as a + * syntax error and surface as a 500, and `limit: 1e9` is an unbounded scan + * bounded only by the profile's own settings. + * + * The ceiling is the part that has to be here rather than in the caller: the + * web client's own input schema checks positivity but has no upper bound, and + * the internal API is reachable by any authenticated client regardless. + */ +const RowLimit = Schema.Number.check( + Schema.isInt(), + Schema.isGreaterThan(0), + Schema.isLessThanOrEqualTo(1000), +).pipe( + Schema.annotate({ + identifier: "RowLimit", + description: "Maximum rows to return: a whole number between 1 and 1000.", + }), +) + // Dedicated endpoint schemas /** Shared primitives for filtered list/facet endpoints. */ @@ -1746,9 +1771,11 @@ export class ProductEventNamesResponse extends Schema.Class( "ProductEventsForTraceRequest", @@ -1756,8 +1783,8 @@ export class ProductEventsForTraceRequest extends Schema.Class( @@ -1787,8 +1814,8 @@ export class ProductEventTraceSamplesRequest extends Schema.Class( diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index 6e98a0339..c02dcdc00 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2212,10 +2212,20 @@ export const productEvents = defineDatasource("product_events", { * filter on it, and a `Map` lookup on this table reads the whole map per * row — the cost `product_events` was split out of `session_events` to * avoid. Last in the schema because `ALTER TABLE … ADD COLUMN` appends. + * + * NO `jsonPath`, deliberately, and load-bearing: these two are written + * only by `product_events_traces_mv` and its backfill, never by an + * ingested NDJSON line. `scripts/generate-clickhouse-insert-mappings.ts` + * skips columns whose path is null, so the Rust gateway's + * `INSERT INTO product_events (…)` does not name them — which is what lets + * migration 0024 stay `requiredForIngest: false`. Give them a path and the + * gateway starts naming columns that a BYO cluster stamped below 24 does + * not have, and every `/v1/events` batch for those orgs is rejected and + * dropped. Same shape as `service_usage`, whose columns are MV-only too. */ - TraceId: column(t.string().default(""), { jsonPath: "$.trace_id" }), + TraceId: t.string().default(""), /** The annotated span within {@link TraceId}. `''` on non-trace rows. */ - SpanId: column(t.string().default(""), { jsonPath: "$.span_id" }), + SpanId: t.string().default(""), }, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index c9f8d05c2..6352392f3 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -190,9 +190,7 @@ export { productEventNamesQuery, productEventNamesRowSchema, productEventsForTraceQuery, - productEventForTraceRowSchema, productEventTraceSamplesQuery, - productEventTraceSampleRowSchema, ProductEventsFunnelError, FUNNEL_MAX_STEPS, FUNNEL_BREAKDOWN_MAX_GROUPS, diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index d3490338b..18480fc26 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -710,18 +710,26 @@ export function productEventNamesQuery( // `idx_trace_id` bloom filter — a `Map` read would take the whole map per row on // the one table that exists to avoid exactly that. -export const productEventForTraceRowSchema = Schema.Struct({ - timestamp: Schema.String, - eventName: Schema.String, - spanId: Schema.String, - serviceName: Schema.String, - userId: Schema.String, - groupId: Schema.String, - visitorId: Schema.String, - sessionId: Schema.String, - attributes: Schema.Record(Schema.String, Schema.String), -}) -export type ProductEventForTraceOutput = typeof productEventForTraceRowSchema.Type +// No declared `rowSchema` on either query below. Every projected column is a +// plain String or the `Map(String, String)` the builder already derives as +// `Schema.Record(Schema.String, Schema.String)`, so a declared copy would be +// byte-identical to the derived one — which is exactly the class of schema the +// 2026-08 sweep deleted 22 of. A declared schema earns its place only when it +// NARROWS (a literal union, a null-collapsing transform) or when derivation is +// wrong (LEFT-JOIN nullability). Neither applies here, and an exported copy +// nothing passes to `compile` is a contract that silently drifts. + +export interface ProductEventForTraceOutput { + readonly timestamp: string + readonly eventName: string + readonly spanId: string + readonly serviceName: string + readonly userId: string + readonly groupId: string + readonly visitorId: string + readonly sessionId: string + readonly attributes: Record +} export interface ProductEventsForTraceOpts { /** Default 50 — a single trace producing more than this is pathological. */ @@ -769,15 +777,14 @@ export function productEventsForTraceQuery( .format("JSON") } -export const productEventTraceSampleRowSchema = Schema.Struct({ - traceId: Schema.String, - spanId: Schema.String, - timestamp: Schema.String, - serviceName: Schema.String, - userId: Schema.String, - visitorId: Schema.String, -}) -export type ProductEventTraceSampleOutput = typeof productEventTraceSampleRowSchema.Type +export interface ProductEventTraceSampleOutput { + readonly traceId: string + readonly spanId: string + readonly timestamp: string + readonly serviceName: string + readonly userId: string + readonly visitorId: string +} export interface ProductEventTraceSamplesOpts { /** Default 20. */ From 4f5d457dc322ea19ba06958bae20a88f05328997 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 1 Sep 2026 14:34:46 +0200 Subject: [PATCH 3/5] fix(cli): commit the regenerated local-schema projectRevision header MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Dropping the jsonPath from product_events.TraceId/SpanId changed the project revision, so both local-schema.sql and its v14 snapshot carry a new header line. The DDL is byte-identical — which is why the local schema identity hash did not move — but clickhouse:schema:check compares the whole file, and the regenerated versions were left unstaged in the previous commit. --- apps/cli/src/server/schema/local-schema-v14.sql | 2 +- apps/cli/src/server/schema/local-schema.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/server/schema/local-schema-v14.sql b/apps/cli/src/server/schema/local-schema-v14.sql index c8cab6145..805c5f6b1 100644 --- a/apps/cli/src/server/schema/local-schema-v14.sql +++ b/apps/cli/src/server/schema/local-schema-v14.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb +-- projectRevision: 5b176340a54f37477fae211fa837c6badd494e1a7cf48c9ea923b58c28783fdb -- localSchemaVersion: 14 CREATE TABLE IF NOT EXISTS alert_checks ( diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index c8cab6145..805c5f6b1 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,6 +1,6 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: 09c55b9a8ea0014b91c82928e78a4faca1df1d497103eab0431241890b2899fb +-- projectRevision: 5b176340a54f37477fae211fa837c6badd494e1a7cf48c9ea923b58c28783fdb -- localSchemaVersion: 14 CREATE TABLE IF NOT EXISTS alert_checks ( From 5175315a2797ec15e72d1b9e959c2390e5f70808 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 1 Sep 2026 14:36:19 +0200 Subject: [PATCH 4/5] fix(domain): regenerate the anticipated-error identifier list MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Carried in from main, which is red at 59db862b97 for this reason: IngestAttributeMappingForbiddenError was added to the ingest-attribute-mappings HTTP schema without rerunning `gen:anticipated-errors`, so the checked-in literal list (115) no longer matched what reflection derives (116) and anticipated-errors.test.ts failed. Not this branch's bug — it is inherited because CI builds the merge commit — but the merge cannot go green without it. The list is generated, so this is purely the output of `bun run --cwd packages/domain gen:anticipated-errors`. Consequence of the gap, for the record: the identifier gates whether a span failing entirely with that error records as OTLP status Ok rather than Error. Missing from the list, a plain 403 from that route would have counted as a real error in error_events_mv. --- packages/domain/src/generated/anticipated-error-identifiers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index 32592a5fa..add5dd953 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -47,6 +47,7 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/ErrorIssuePullRequestNotFoundError", "@maple/http/errors/ErrorIssueTransitionError", "@maple/http/errors/ErrorValidationError", + "@maple/http/errors/IngestAttributeMappingForbiddenError", "@maple/http/errors/IngestAttributeMappingNotFoundError", "@maple/http/errors/IngestAttributeMappingValidationError", "@maple/http/errors/IntegrationsForbiddenError", From be27da60b4f28026d5541780297d8f7f36da6bb5 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Tue, 1 Sep 2026 14:46:34 +0200 Subject: [PATCH 5/5] fix(lint): clear the three effect-lint errors carried in from main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second half of main's breakage at 59db862b97 (the first was the anticipated- error list). Main is red for exactly `TypeScript (effect-lint)` and `TypeScript (test-packages)`; CI builds the merge commit, so this branch inherits both and cannot go green without them. All three sites landed with #717-#719 and none are this branch's code. One is a real violation: - PlanetScaleConnectionService: two consecutive `catchTag` calls collapse into a single `catchTags`, which is what the rule asks for and what the rest of the repo does. Two are the heuristic firing on correct code, suppressed with a reason rather than "fixed" into something worse: - ElectricClient `shape` is Electric's own domain term — a shape is its unit of subscription — and the value is written to the `maple.electric.shape` span attribute under that exact name. Renaming it to satisfy no-shape-in-symbol-names would make the code describe Electric less accurately. Suppressed across the function rather than at one line, since the parameter and its use both trip it. - WarehouseQueryService's two fetch test doubles use `as unknown as typeof fetch` because `typeof fetch`'s overload set is not satisfiable by a bare async function. The narrowing is local to a test. Same shape as the existing anti-slop suppressions in the v2 OpenAPI contract tests. Directives are placed on the line immediately above the offending code with the prose above them — an `oxlint-disable-next-line` whose justification wraps onto a second comment line targets that comment, not the code, and reports as an unused directive while the original error stands. Verified: full `bun run lint` clean, apps/api WarehouseQueryService 36 passed, apps/electric-sync 76 passed, both packages typecheck clean. --- .../integrations/PlanetScaleConnectionService.ts | 15 +++++++-------- .../warehouse/WarehouseQueryService.test.ts | 6 ++++++ apps/electric-sync/src/electric/ElectricClient.ts | 6 ++++++ 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/apps/api/src/services/integrations/PlanetScaleConnectionService.ts b/apps/api/src/services/integrations/PlanetScaleConnectionService.ts index 9c8fb2e37..fec54af2a 100644 --- a/apps/api/src/services/integrations/PlanetScaleConnectionService.ts +++ b/apps/api/src/services/integrations/PlanetScaleConnectionService.ts @@ -733,14 +733,13 @@ export class PlanetScaleConnectionService extends Context.Service< const target = yield* selectManagedTarget(connection) if (target !== null && target.managedBy === managedByForConnection(connection.id)) { yield* scrapeTargetsService.delete(orgId, target.id, { allowManaged: true }).pipe( - Effect.catchTag("@maple/http/errors/ScrapeTargetNotFoundError", () => - Effect.annotateCurrentSpan("maple.planetscale.disconnect_target_missing", true), - ), - // `allowManaged` is the only thing delete validates, so this - // branch is unreachable — a reachable one is a bug, not a 400. - Effect.catchTag("@maple/http/errors/ScrapeTargetValidationError", (error) => - Effect.die(error), - ), + Effect.catchTags({ + "@maple/http/errors/ScrapeTargetNotFoundError": () => + Effect.annotateCurrentSpan("maple.planetscale.disconnect_target_missing", true), + // `allowManaged` is the only thing delete validates, so this + // branch is unreachable — a reachable one is a bug, not a 400. + "@maple/http/errors/ScrapeTargetValidationError": (error) => Effect.die(error), + }), ) } diff --git a/apps/api/src/services/warehouse/WarehouseQueryService.test.ts b/apps/api/src/services/warehouse/WarehouseQueryService.test.ts index 555d2d7c2..bc1221200 100644 --- a/apps/api/src/services/warehouse/WarehouseQueryService.test.ts +++ b/apps/api/src/services/warehouse/WarehouseQueryService.test.ts @@ -983,6 +983,9 @@ describe("BYO ClickHouse redirect refusal", () => { it("refuses a 3xx from the query endpoint and never follows the Location", async () => { const seen: RequestInit[] = [] + // A fetch test double cannot satisfy `typeof fetch`'s full overload set; the double + // assert is the standard way to build one, and the narrowing is local to this test. + // oxlint-disable-next-line anti-slop/no-chained-type-assertions -- fetch test double const requestFetch = (async (_input: unknown, init: RequestInit) => { seen.push(init) return new Response("", { status: 307, headers: { location: "http://169.254.169.254/" } }) @@ -1007,6 +1010,9 @@ describe("BYO ClickHouse redirect refusal", () => { }) it("passes an ordinary 2xx response through untouched", async () => { + // Same fetch test double as above; `typeof fetch`'s overloads are not satisfiable by a + // bare async function. + // oxlint-disable-next-line anti-slop/no-chained-type-assertions -- fetch test double const requestFetch = (async () => new Response('{"n":1}\n', { status: 200 })) as unknown as typeof fetch const response = await __testables.redirectRefusingFetch(requestFetch)("https://ch.example.com") diff --git a/apps/electric-sync/src/electric/ElectricClient.ts b/apps/electric-sync/src/electric/ElectricClient.ts index fdb1d9432..0d06c7692 100644 --- a/apps/electric-sync/src/electric/ElectricClient.ts +++ b/apps/electric-sync/src/electric/ElectricClient.ts @@ -68,6 +68,11 @@ export const describeUpstreamFailure = (error: unknown): string => { * * Exported so the omission is asserted directly, not inferred from a span. */ +// `shape` is Electric's own domain term — a shape is its unit of subscription — not a structural +// placeholder, and the value is written to the `maple.electric.shape` span attribute under that +// exact name. Renaming it to satisfy the heuristic would make the code describe Electric less +// accurately, so the rule is suppressed across the function body instead. +/* oxlint-disable anti-slop/no-shape-in-symbol-names */ export const sanitizedUpstreamAttributes = (upstreamUrl: string, shape: string): Record => { const url = new URL(upstreamUrl) return { @@ -79,6 +84,7 @@ export const sanitizedUpstreamAttributes = (upstreamUrl: string, shape: string): "maple.electric.shape": shape, } } +/* oxlint-enable anti-slop/no-shape-in-symbol-names */ const isLiveRequest = (clientParams: URLSearchParams): boolean => { const live = clientParams.get("live")