diff --git a/apps/api/src/routes/internal/query-engine.http.ts b/apps/api/src/routes/internal/query-engine.http.ts index 2c2cd8213..d749b0051 100644 --- a/apps/api/src/routes/internal/query-engine.http.ts +++ b/apps/api/src/routes/internal/query-engine.http.ts @@ -77,6 +77,8 @@ import { ProductEventsFunnelResponse, ProductEventsFunnelBreakdownResponse, ProductEventNamesResponse, + ProductEventsForTraceResponse, + ProductEventTraceSamplesResponse, CommitSha, FingerprintHash, ServiceName, @@ -2066,6 +2068,45 @@ export const HttpQueryEngineLive = HttpApiBuilder.group(MapleInternalApi, "query }) }), ) + // Both directions of the trace ↔ product-event link. + .handle("productEventsForTrace", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventsForTrace, tenant, payload) + return new ProductEventsForTraceResponse({ + data: rows.map((row) => ({ + timestamp: String(row.timestamp), + eventName: String(row.eventName), + spanId: String(row.spanId), + serviceName: String(row.serviceName), + userId: String(row.userId), + groupId: String(row.groupId), + visitorId: String(row.visitorId), + sessionId: String(row.sessionId), + // Already decoded as Record by the derived row + // schema — a non-string value fails that decode long before it + // reaches here, so there is nothing left to coerce. + attributes: row.attributes, + })), + }) + }), + ) + .handle("productEventTraceSamples", ({ payload }) => + Effect.gen(function* () { + const tenant = yield* CurrentTenant.Context + const rows = yield* runQuery(Queries.productEventTraceSamples, tenant, payload) + return new ProductEventTraceSamplesResponse({ + data: rows.map((row) => ({ + traceId: String(row.traceId), + spanId: String(row.spanId), + timestamp: String(row.timestamp), + serviceName: String(row.serviceName), + userId: String(row.userId), + visitorId: String(row.visitorId), + })), + }) + }), + ) .handle("executeRawSql", ({ payload }) => Effect.gen(function* () { const tenant = yield* CurrentTenant.Context diff --git a/apps/cli/src/server/local-schema-history.ts b/apps/cli/src/server/local-schema-history.ts index 403e138d3..b532a4880 100644 --- a/apps/cli/src/server/local-schema-history.ts +++ b/apps/cli/src/server/local-schema-history.ts @@ -163,4 +163,17 @@ export const LOCAL_SCHEMA_HISTORY: ReadonlyArray = Obje manifestDigest: "65f0bc9e91171fbefd4452373ad15f8139b56111ffff27f65ffa4a09ba82cdb2", projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", }), + Object.freeze({ + // TODO(v16): what changed, whether any part is rewritten or any row + // moves, and what this edge does NOT backfill. + // + // projectRevision is carried forward deliberately — it is a hardcoded + // constant that no longer tracks the generator's header, and the identity + // this gate compares is the fingerprint/digest pair. + version: 16, + fingerprint: "3cfe5f649a11853a", + digest: "3cfe5f649a11853ae87021dfb611b0ef76cbd6be4a597d247fb2dff4e717c59c", + manifestDigest: "91ef0f362cc378f3e83dec0cd40065506b36d0447487dab1d0e6887e5c660bad", + projectRevision: "ed74788ef292834069e0ea6ee3b22d68fc604fb66cb54d2d551db67ce8d20b3a", + }), ] as const) diff --git a/apps/cli/src/server/local-schema-version.ts b/apps/cli/src/server/local-schema-version.ts index de7a1ff68..f11aae74c 100644 --- a/apps/cli/src/server/local-schema-version.ts +++ b/apps/cli/src/server/local-schema-version.ts @@ -1,4 +1,4 @@ // Increment this value for every structural change to the generated local // schema. The compatibility manifest and migration registry must be updated in // the same change before a new value can ship. -export const LOCAL_SCHEMA_VERSION = 15 as const +export const LOCAL_SCHEMA_VERSION = 16 as const diff --git a/apps/cli/src/server/local-store-migrations.ts b/apps/cli/src/server/local-store-migrations.ts index fce8947d2..f2804c07f 100644 --- a/apps/cli/src/server/local-store-migrations.ts +++ b/apps/cli/src/server/local-store-migrations.ts @@ -51,6 +51,7 @@ import { v11ToV12ServiceMapEdgeQuantilesModule } from "./local-store-migrations/ import { v12ToV13ServiceOperationsDiscriminatorsModule } from "./local-store-migrations/v12-to-v13-service-operations-discriminators" import { v13ToV14AiTraceIndexModule } from "./local-store-migrations/v13-to-v14-ai-trace-index" import { v14ToV15CommitShaVcsRevisionModule } from "./local-store-migrations/v14-to-v15-commit-sha-vcs-revision" +import { v15ToV16ProductEventsFromTracesModule } from "./local-store-migrations/v15-to-v16-product-events-from-traces" import type { AnyLocalStoreMigrationModule, LocalStoreMigration, @@ -123,6 +124,7 @@ export const localStoreMigrations: ReadonlyArray = v12ToV13ServiceOperationsDiscriminatorsModule, v13ToV14AiTraceIndexModule, v14ToV15CommitShaVcsRevisionModule, + v15ToV16ProductEventsFromTracesModule, ] export const validateMigrationRegistry = ( diff --git a/apps/cli/src/server/local-store-migrations/v15-to-v16-product-events-from-traces.ts b/apps/cli/src/server/local-store-migrations/v15-to-v16-product-events-from-traces.ts new file mode 100644 index 000000000..d1799d748 --- /dev/null +++ b/apps/cli/src/server/local-store-migrations/v15-to-v16-product-events-from-traces.ts @@ -0,0 +1,436 @@ +// SAFETY-FILE: JSON rows here come from fixed internal formats and are validated before domain use. +import { cloneStoreForStaging } from "./journal-codecs" +import { resolve } from "node:path" +import { RAW_TELEMETRY_TTL_COLUMNS, readRawTelemetryRetentionDays, type Chdb } from "../chdb" +import type { + LocalStoreMigrationModule, + MigrationModuleContext, + MigrationOperation, + StateDispositionEntry, +} from "../local-store-migration-module" +import { withRawTelemetryRetentionFloor } from "../schema-manifest" +import { + LOCAL_SCHEMA_V15, + LOCAL_SCHEMA_V15_MANIFEST, + LOCAL_SCHEMA_V15_SQL, + LOCAL_SCHEMA_V16, + LOCAL_SCHEMA_V16_MANIFEST, + LOCAL_SCHEMA_V16_SQL, +} from "../schema-identity" +import { assertPhysicalSchema } from "../schema-physical" + +const RAW_TABLES = RAW_TELEMETRY_TTL_COLUMNS.map(([table]) => table) + +const MODULE_ID = "local-0015-to-0016-product-events-from-traces" as const + +/** + * Frozen copy of ClickHouse migration 0026's trace projection. Frozen for the + * reason every edge in this directory freezes its SQL: this module describes one + * step in history, and importing the live projection would silently rewrite what + * v15 -> v16 did the next time it changes. + */ +const PRODUCT_EVENTS_TRACE_COLUMNS = [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + "TraceId", + "SpanId", +].join(", ") + +const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + CAST(SpanAttributes, 'Map(String, String)') AS Attributes, + TraceId, + SpanId` + +const PRODUCT_EVENTS_TRACE_FILTER = "SpanAttributes['maple.product_event.name'] != ''" + +/** + * The local mirror of ClickHouse migration 0026. + * + * `product_events` gains `TraceId`/`SpanId` (`DEFAULT ''`) and a bloom filter on + * `TraceId`, and a second view — `product_events_traces_mv` — starts projecting + * spans the user annotated in their own code (`maple.product_event.name`) into + * the table. The trace id is what makes a product event and the trace that + * produced it navigable from either side. + * + * THE TRACE HALF IS BACKFILLED. Every annotated span still inside the local + * store's raw `traces` retention is re-projected, so an `EventName` a user just + * added to their code has history the moment they upgrade rather than only going + * forward — bounded by that retention (30 days by default) against + * `product_events`' own 365, so the older part of the window stays empty and + * accrues from here. + * + * `product_events_mv` — the browser feed — is dropped and recreated too. Its + * SELECT was frozen at 15 columns and the table now has 17; the two new ones + * default to `''`, which is the right value for a browser row, so recreating it + * repairs nothing and instead keeps the view's text and the table's schema + * describing the same thing. + * + * Every statement is idempotent — `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT + * EXISTS`, and a `DELETE` scoped to the backfill's own source window — so a + * resume after a crash between them lands in the same place. + */ + +interface V15ToV16State { + readonly module: typeof MODULE_ID + readonly version: 1 + readonly rawRows: Readonly> + readonly productEventRows: ProductEventRowCounts + readonly retentionDays?: number +} + +/** + * What `product_events` held before the edge, and what the backfill is expected + * to add. + * + * `existing` is every row already in the table — at v15 none of them can be a + * trace row, because the source did not exist — and is re-checked as an + * EQUALITY: the backfill must add rows, never disturb one. `expectedTrace` is + * counted from `traces` under the very filter the backfill uses, so verify + * compares against the exact number rather than a lower bound, and an + * `INSERT … SELECT` that half-completed shows up as a mismatch instead of + * passing as "some rows arrived". + */ +interface ProductEventRowCounts { + readonly existing: string + readonly expectedTrace: string +} + +interface V15ToV16Progress { + readonly installed: true +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value) + +const isCount = (value: unknown): value is string => typeof value === "string" && /^\d+$/.test(value) + +const decodeCounts = (value: unknown): Readonly> => { + if (!isRecord(value)) throw new Error("v15 -> v16 rawRows must be an object") + const counts: Record = {} + for (const table of RAW_TABLES) { + const count = value[table] + if (!isCount(count)) throw new Error(`v15 -> v16 rawRows.${table} must be an unsigned decimal string`) + counts[table] = count + } + if (Object.keys(value).some((table) => !RAW_TABLES.includes(table as (typeof RAW_TABLES)[number]))) + throw new Error("v15 -> v16 rawRows contains an unknown table") + return counts +} + +const decodeProductEventRows = (value: unknown): ProductEventRowCounts => { + if (!isRecord(value)) throw new Error("v15 -> v16 productEventRows must be an object") + if (Object.keys(value).some((key) => key !== "existing" && key !== "expectedTrace")) + throw new Error("v15 -> v16 productEventRows contains an unknown field") + if (!isCount(value.existing)) + throw new Error("v15 -> v16 productEventRows.existing must be an unsigned decimal string") + if (!isCount(value.expectedTrace)) + throw new Error("v15 -> v16 productEventRows.expectedTrace must be an unsigned decimal string") + return { existing: value.existing, expectedTrace: value.expectedTrace } +} + +const decodeState = (value: unknown): V15ToV16State => { + if (!isRecord(value)) throw new Error("v15 -> v16 state must be an object") + const allowed = new Set(["module", "version", "rawRows", "productEventRows", "retentionDays"]) + if (Object.keys(value).some((key) => !allowed.has(key))) + throw new Error("v15 -> v16 state contains an unknown field") + if (value.module !== MODULE_ID || value.version !== 1) + throw new Error("v15 -> v16 state has an unsupported module or version") + if ( + value.retentionDays !== undefined && + (typeof value.retentionDays !== "number" || !Number.isSafeInteger(value.retentionDays)) + ) + throw new Error("v15 -> v16 retentionDays must be an integer") + return { + module: MODULE_ID, + version: 1, + rawRows: decodeCounts(value.rawRows), + productEventRows: decodeProductEventRows(value.productEventRows), + ...(!(value.retentionDays === undefined) ? { retentionDays: value.retentionDays } : undefined), + } +} + +const decodeProgress = (value: unknown): V15ToV16Progress | undefined => { + if (value === undefined) return undefined + if (!isRecord(value) || Object.keys(value).some((key) => key !== "installed") || value.installed !== true) + throw new Error("v15 -> v16 progress is invalid") + return { installed: true } +} + +const parseJsonEachRow = (value: string): A[] => + value + .split("\n") + .map((line) => line.trim()) + .filter((line) => line.length > 0) + .map((line) => JSON.parse(line) as A) + +const rawRowCounts = (db: Chdb): Readonly> => { + const quotedTables = RAW_TABLES.map((table) => `'${table}'`).join(", ") + const rows = parseJsonEachRow<{ table: string; rowCount: string }>( + db.query( + `SELECT table, toString(sum(rows)) AS rowCount FROM system.parts WHERE database = 'default' AND active = 1 AND table IN (${quotedTables}) GROUP BY table`, + ), + ) + const byTable = new Map(rows.map((row) => [row.table, row.rowCount])) + return Object.fromEntries(RAW_TABLES.map((table) => [table, byTable.get(table) ?? "0"])) +} + +const scalarCount = (db: Chdb, sql: string): string => { + const rows = parseJsonEachRow<{ count: string }>(db.query(sql)) + const count = rows[0]?.count + if (!isCount(count)) throw new Error(`v15 -> v16 count query returned no row: ${sql}`) + return count +} + +const productEventRowCounts = (db: Chdb): ProductEventRowCounts => ({ + existing: scalarCount(db, "SELECT toString(count()) AS count FROM product_events"), + expectedTrace: scalarCount( + db, + `SELECT toString(count()) AS count FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ), +}) + +const expectedManifest = (manifest: typeof LOCAL_SCHEMA_V15_MANIFEST, retentionDays: number | undefined) => + retentionDays === undefined + ? manifest + : withRawTelemetryRetentionFloor(manifest, RAW_TABLES, retentionDays) + +const preflight = async (context: MigrationModuleContext): Promise => { + await context.ensureCapacity() + const retentionDays = readRawTelemetryRetentionDays(context.dataDir) + const { rawRows, productEventRows } = await context.openSource( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V15_MANIFEST, retentionDays)) + return { rawRows: rawRowCounts(db), productEventRows: productEventRowCounts(db) } + }, + { schemaSql: LOCAL_SCHEMA_V15_SQL, bootstrapSchema: false }, + ) + return { + module: MODULE_ID, + version: 1, + rawRows, + productEventRows, + ...(!(retentionDays === undefined) ? { retentionDays } : undefined), + } +} + +const prepareTarget = async ( + context: MigrationModuleContext, + state: V15ToV16State, +): Promise => { + await context.closeStores() + const source = resolve(context.sourceDataDir) + const target = resolve(context.targetDataDir) + if (source !== target) { + await cloneStoreForStaging(source, target) + } + return state +} + +/** + * The columns, the index AND both view drops happen before the v16 bootstrap, in + * the v15-schema block. The ordering is load-bearing in both directions: + * + * - `CREATE TABLE IF NOT EXISTS` is a no-op against the cloned store, so the + * v15 snapshot alone leaves `product_events` without its two new columns. + * - `CREATE MATERIALIZED VIEW IF NOT EXISTS` is equally a no-op while the old + * view still exists, and a view's SELECT is frozen at creation. Dropping the + * views AFTER the bootstrap would delete them outright — the bootstrap has + * already skipped them and nothing recreates them. + * + * The backfill then runs in the v16 block, after the bootstrap has created both + * views. It writes `product_events` directly and the views read `session_events` + * and `traces`, so nothing double-fires. + */ +const apply = async (context: MigrationModuleContext): Promise => { + await context.openTarget( + (db) => { + db.exec("ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''") + db.exec("ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''") + db.exec( + "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + ) + db.exec("DROP VIEW IF EXISTS product_events_traces_mv") + db.exec("DROP VIEW IF EXISTS product_events_mv") + }, + { schemaSql: LOCAL_SCHEMA_V15_SQL, bootstrapSchema: false }, + ) + return context.openTarget( + (db) => { + // Scoped to the backfill's own source window, matching migration 0026: + // `traces` keeps 30 days and `product_events` 365, so an unbounded + // delete would destroy trace rows a late re-run can no longer rebuild. + // A v15 store has no trace rows at all — the source did not exist — so + // this is a no-op on the edge it actually runs on, and correct on any + // resume or re-apply that finds some. + db.exec( + "DELETE FROM product_events WHERE Source = 'trace' AND Timestamp >= (SELECT min(Timestamp) FROM traces)", + ) + db.exec( + `INSERT INTO product_events (${PRODUCT_EVENTS_TRACE_COLUMNS}) SELECT ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} FROM traces WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ) + return { installed: true } as const + }, + { schemaSql: LOCAL_SCHEMA_V16_SQL, bootstrapSchema: true }, + ) +} + +const verify = async ( + context: MigrationModuleContext, + state: V15ToV16State, + _progress: V15ToV16Progress, +): Promise => { + await context.openTarget( + (db) => { + assertPhysicalSchema(db, expectedManifest(LOCAL_SCHEMA_V16_MANIFEST, state.retentionDays)) + const targetRows = rawRowCounts(db) + for (const table of RAW_TABLES) { + if (targetRows[table] !== state.rawRows[table]) + throw new Error(`v15 -> v16 raw telemetry verification failed for ${table}`) + } + // Split rather than a single total: an equality on the pre-existing rows + // is what proves the backfill only ADDED, and an equality on the trace + // rows is what proves it added all of them. A total would let one error + // cancel the other out. + const existing = scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source != 'trace'", + ) + if (existing !== state.productEventRows.existing) + throw new Error( + `v15 -> v16 pre-existing product_events row count changed: expected ${state.productEventRows.existing}, found ${existing}`, + ) + const backfilled = scalarCount( + db, + "SELECT toString(count()) AS count FROM product_events WHERE Source = 'trace'", + ) + if (backfilled !== state.productEventRows.expectedTrace) + throw new Error( + `v15 -> v16 backfilled trace product_events row count mismatch: expected ${state.productEventRows.expectedTrace}, found ${backfilled}`, + ) + }, + { schemaSql: LOCAL_SCHEMA_V16_SQL, bootstrapSchema: false }, + ) +} + +const operations: ReadonlyArray = [ + { + id: "clone-v15-store", + description: "Clone the stopped v15 store into the staged migration target", + requiresQuiescence: true, + phase: "target-created", + }, + { + id: "add-product-event-trace-columns", + description: + "Add TraceId and SpanId to product_events, plus the TraceId bloom filter the trace lookup prunes on", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "backfill-annotated-spans", + description: + "Project every retained span carrying maple.product_event.name into product_events as a Source='trace' row", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "recreate-product-event-views", + description: + "Recreate product_events_mv and create product_events_traces_mv so new rows carry TraceId and annotated spans keep arriving", + requiresQuiescence: true, + phase: "copying", + }, + { + id: "verify-v16-schema", + description: + "Verify the v16 physical schema, retained raw telemetry counts, and that the backfill added exactly the annotated spans and disturbed no existing row", + requiresQuiescence: true, + phase: "copy-verified", + }, +] + +const dispositions: ReadonlyArray = [ + { + name: "local store", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: "The clean stopped v15 store is cloned byte-for-byte before any DDL runs.", + }, + { + name: "traces", + classification: "authoritative", + disposition: "preserve-exact", + guarantee: + "Read-only source of the backfill; the row count is verified unchanged alongside every other raw telemetry table.", + }, + { + name: "product_events (browser, server and mobile rows)", + classification: "derived", + disposition: "preserve-exact", + guarantee: + "Two columns are added as metadata-only defaults, no part is rewritten, and the count of rows whose Source is not 'trace' is verified unchanged after the backfill. Counts, not contents: the byte-level claim rests on ADD COLUMN being metadata-only, which this edge does not independently verify.", + }, + { + // Fully rebuilt within the raw window, then accrued: unlike the service + // operations edges there IS a source to re-project from, so the shorter + // retention is the only bound. + name: "product_events (trace rows)", + classification: "derived", + disposition: "rebuild-within-retention-horizon", + guarantee: + "Every annotated span still inside raw traces retention is re-projected, and the resulting row count is verified to equal the count of matching spans. Annotated spans older than that window are gone from traces and cannot be rebuilt; the table accrues them from the migration forward.", + preservationInterval: "the raw traces retention window", + // The schema default. A store running a custom raw-telemetry floor (see + // `readRawTelemetryRetentionDays`) keeps more or less than this, and the + // backfill follows the store rather than this number. + sourceRetentionDays: 30, + targetRetentionDays: 365, + }, +] + +export const v15ToV16ProductEventsFromTracesModule: LocalStoreMigrationModule< + V15ToV16State, + V15ToV16Progress +> = { + id: MODULE_ID, + moduleVersion: 1, + description: + "Add TraceId/SpanId to product_events and project spans annotated with maple.product_event.name into it, backfilled from retained traces", + from: LOCAL_SCHEMA_V15, + to: LOCAL_SCHEMA_V16, + operations, + dispositions, + decodeState, + decodeProgress, + preflight, + prepareTarget, + apply, + verify, + recover: async (_context, state, progress) => ({ state, progress }), +} diff --git a/apps/cli/src/server/schema-identity.ts b/apps/cli/src/server/schema-identity.ts index 77b1a3322..5c413fadd 100644 --- a/apps/cli/src/server/schema-identity.ts +++ b/apps/cli/src/server/schema-identity.ts @@ -14,6 +14,7 @@ import schemaV12Sql from "./schema/local-schema-v12.sql" with { type: "text" } import schemaV13Sql from "./schema/local-schema-v13.sql" with { type: "text" } import schemaV14Sql from "./schema/local-schema-v14.sql" with { type: "text" } import schemaV15Sql from "./schema/local-schema-v15.sql" with { type: "text" } +import schemaV16Sql from "./schema/local-schema-v16.sql" with { type: "text" } import { schemaDigest as digestSchema, schemaFingerprint as fingerprintSchema } from "./store-version" import { buildLocalSchemaManifest, type LocalSchemaManifest } from "./schema-manifest" import { LOCAL_SCHEMA_VERSION } from "./local-schema-version" @@ -73,6 +74,7 @@ const SNAPSHOT_SQL: ReadonlyArray = [ schemaV13Sql, schemaV14Sql, schemaV15Sql, + schemaV16Sql, ] export interface LocalSchemaSnapshot { @@ -127,6 +129,8 @@ export const LOCAL_SCHEMA_V14_SQL = snapshotAt(14).sql export const LOCAL_SCHEMA_V14_MANIFEST = snapshotAt(14).manifest export const LOCAL_SCHEMA_V15_SQL = snapshotAt(15).sql export const LOCAL_SCHEMA_V15_MANIFEST = snapshotAt(15).manifest +export const LOCAL_SCHEMA_V16_SQL = snapshotAt(16).sql +export const LOCAL_SCHEMA_V16_MANIFEST = snapshotAt(16).manifest export interface LocalSchemaIdentity { readonly version: number @@ -172,6 +176,7 @@ export const LOCAL_SCHEMA_V12 = identityAt(12) export const LOCAL_SCHEMA_V13 = identityAt(13) export const LOCAL_SCHEMA_V14 = identityAt(14) export const LOCAL_SCHEMA_V15 = identityAt(15) +export const LOCAL_SCHEMA_V16 = identityAt(16) export const CURRENT_LOCAL_SCHEMA: LocalSchemaIdentity = Object.freeze({ version: LOCAL_SCHEMA_VERSION, diff --git a/apps/cli/src/server/schema/local-inserts.json b/apps/cli/src/server/schema/local-inserts.json index 4866fb8f1..1d7377faa 100644 --- a/apps/cli/src/server/schema/local-inserts.json +++ b/apps/cli/src/server/schema/local-inserts.json @@ -1,5 +1,5 @@ { - "projectRevision": "bf3419c18e581ffab5fa4e24aa56f421ac9c3d5a1ce734596747f2331d7d2ae0", + "projectRevision": "4545ddafc64d7bd0234a65c2442cb615373026c6fcf1d2f4b3f0f0e6b5ca88d9", "orgPlaceholder": "__ORG__", "datasources": { "traces": { diff --git a/apps/cli/src/server/schema/local-schema-v16.sql b/apps/cli/src/server/schema/local-schema-v16.sql new file mode 100644 index 000000000..0bb35482e --- /dev/null +++ b/apps/cli/src/server/schema/local-schema-v16.sql @@ -0,0 +1,1956 @@ +-- This file is generated by scripts/generate-clickhouse-schema-sql.ts +-- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. +-- projectRevision: 4545ddafc64d7bd0234a65c2442cb615373026c6fcf1d2f4b3f0f0e6b5ca88d9 +-- localSchemaVersion: 16 + +CREATE TABLE IF NOT EXISTS ai_trace_index ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SessionId String, + VendorId LowCardinality(String), + ServiceName LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS alert_checks ( + OrgId LowCardinality(String), + RuleId String, + GroupKey String, + Timestamp DateTime64(3), + Status LowCardinality(String), + SignalType LowCardinality(String), + Comparator LowCardinality(String), + Threshold Float64, + ObservedValue Nullable(Float64), + SampleCount UInt32, + WindowMinutes UInt16, + WindowStart DateTime64(3), + WindowEnd DateTime64(3), + ConsecutiveBreaches UInt16, + ConsecutiveHealthy UInt16, + IncidentId Nullable(String), + IncidentTransition LowCardinality(String), + EvaluationDurationMs UInt32, + ErrorMessage Nullable(String), + ErrorCategory LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, RuleId, GroupKey, Timestamp) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS attribute_keys_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, Hour, AttributeKey) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS attribute_values_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + AttributeKey LowCardinality(String), + AttributeValue String, + AttributeScope LowCardinality(String), + UsageCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, AttributeScope, AttributeKey, Hour, AttributeValue) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, FingerprintHash, Timestamp) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_events_by_time ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String DEFAULT '__unset__', + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ExceptionType LowCardinality(String), + ExceptionMessage String, + ExceptionStacktrace String, + TopFrame String, + FingerprintHash UInt64, + StatusMessage String, + Duration UInt64, + ErrorLabel String, + ServiceVersion LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, FingerprintHash) +TTL Timestamp + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS error_fingerprints_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + FingerprintHash UInt64, + ServiceName SimpleAggregateFunction(anyLast, String), + ExceptionType SimpleAggregateFunction(anyLast, String), + ExceptionMessage SimpleAggregateFunction(anyLast, String), + ErrorLabel SimpleAggregateFunction(anyLast, String), + TopFrame SimpleAggregateFunction(anyLast, String), + OccurrenceCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime), + ServiceVersions SimpleAggregateFunction(groupUniqArrayArray, Array(String)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Minute) +ORDER BY (OrgId, Minute, FingerprintHash) +TTL Minute + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS identity_links ( + OrgId LowCardinality(String), + VisitorId String, + UserId String, + FirstSeen SimpleAggregateFunction(min, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY tuple() +ORDER BY (OrgId, VisitorId, UserId) +TTL toDate(FirstSeen) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS logs ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TimestampTime DateTime, + TraceId String, + SpanId String, + TraceFlags UInt8, + SeverityText LowCardinality(String), + SeverityNumber UInt8, + ServiceName LowCardinality(String), + Body String, + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + LogAttributes Map(LowCardinality(String), String), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + LogAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(LogAttributes), mapValues(LogAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_keys mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_log_attr_vals mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_lower_body lower(Body) TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 8 +) +ENGINE = MergeTree +PARTITION BY toDate(TimestampTime) +ORDER BY (OrgId, toStartOfFiveMinutes(Timestamp), ServiceName, Timestamp) +TTL toDate(TimestampTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS logs_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SeverityText LowCardinality(String), + DeploymentEnv LowCardinality(String), + Count SimpleAggregateFunction(sum, UInt64), + SizeBytes SimpleAggregateFunction(sum, UInt64), + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS metric_catalog ( + OrgId LowCardinality(String), + Hour DateTime, + MetricType LowCardinality(String), + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription SimpleAggregateFunction(anyLast, String), + MetricUnit SimpleAggregateFunction(anyLast, String), + IsMonotonic SimpleAggregateFunction(anyLast, UInt8), + DataPointCount SimpleAggregateFunction(sum, UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + LastSeen SimpleAggregateFunction(max, DateTime) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, MetricType, ServiceName, MetricName, Hour) +TTL Hour + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_exponential_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + Scale Int32, + ZeroCount UInt64, + PositiveOffset Int32, + PositiveBucketCounts Array(UInt64), + NegativeOffset Int32, + NegativeBucketCounts Array(UInt64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_gauge ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)) +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_histogram ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Count UInt64, + Sum Float64, + BucketCounts Array(UInt64), + ExplicitBounds Array(Float64), + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + Flags UInt32, + Min Nullable(Float64), + Max Nullable(Float64), + AggregationTemporality Int32 +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS metrics_sum ( + OrgId LowCardinality(String), + ResourceAttributes Map(LowCardinality(String), String), + ResourceSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + MetricDescription LowCardinality(String), + MetricUnit LowCardinality(String), + Attributes Map(LowCardinality(String), String), + StartTimeUnix DateTime64(9), + TimeUnix DateTime64(9), + Value Float64, + Flags UInt32, + ExemplarsTraceId Array(String), + ExemplarsSpanId Array(String), + ExemplarsTimestamp Array(DateTime64(9)), + ExemplarsValue Array(Float64), + ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)), + AggregationTemporality Int32, + IsMonotonic Bool +) +ENGINE = MergeTree +PARTITION BY toDate(TimeUnix) +ORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix)) +TTL toDate(TimeUnix) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS product_events ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + Source LowCardinality(String) DEFAULT 'browser', + SessionId String DEFAULT '', + Seq UInt32 DEFAULT 0, + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + Kind LowCardinality(String), + EventName String, + Host LowCardinality(String) DEFAULT '', + PagePath String DEFAULT '', + Url String DEFAULT '', + ServiceName LowCardinality(String) DEFAULT '', + Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', + INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq) +TTL toDate(Timestamp) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + ParentServerAddress String, + ResolvedTargetService LowCardinality(String), + DeploymentEnv LowCardinality(String) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_external_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + TargetType LowCardinality(String), + TargetSystem LowCardinality(String), + TargetName String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_children ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, ParentSpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64), + DbNamespace LowCardinality(String), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_db_query_shapes_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DbSystem LowCardinality(String), + DeploymentEnv LowCardinality(String), + QueryKey String, + QueryLabel SimpleAggregateFunction(any, String), + SampleStatement SimpleAggregateFunction(any, String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedCount SimpleAggregateFunction(sum, Float64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSumMs SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32), + DbNamespace LowCardinality(String) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, DbSystem, DbNamespace, QueryKey) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount SimpleAggregateFunction(sum, UInt64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + DurationSumMs SimpleAggregateFunction(sum, Float64), + MaxDurationMs SimpleAggregateFunction(max, Float64), + SampledSpanCount SimpleAggregateFunction(sum, UInt64), + UnsampledSpanCount SimpleAggregateFunction(sum, UInt64), + SampleRateSum SimpleAggregateFunction(sum, Float64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, DeploymentEnv, SourceService, TargetService) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_map_edges_hourly_ingest ( + OrgId LowCardinality(String), + Hour DateTime, + SourceService LowCardinality(String), + TargetService String, + DeploymentEnv LowCardinality(String), + CallCount UInt64, + ErrorCount UInt64, + DurationSumMs Float64, + MaxDurationMs Float64, + SampledSpanCount UInt64, + UnsampledSpanCount UInt64, + SampleRateSum Float64 +) +ENGINE = Null; + +CREATE TABLE IF NOT EXISTS service_map_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + TraceId String, + SpanId String, + ParentSpanId String, + ServiceName LowCardinality(String), + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Hour, SpanName) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_operations_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + SpanName String, + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95), UInt64), + ClassifiedSpanCount SimpleAggregateFunction(sum, UInt64), + ServerSpanCount SimpleAggregateFunction(sum, UInt64), + RoutedSpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, DeploymentEnv, Minute, SpanName) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toYYYYMM(Hour) +ORDER BY (OrgId, ServiceName, Hour, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_minutely ( + OrgId LowCardinality(String), + Minute DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + ServiceNamespace LowCardinality(String), + CommitSha LowCardinality(String), + SpanCount SimpleAggregateFunction(sum, UInt64), + EstimatedSpanCount SimpleAggregateFunction(sum, Float64), + ErrorCount SimpleAggregateFunction(sum, UInt64), + EstimatedErrorCount SimpleAggregateFunction(sum, Float64), + DurationSum SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigest(0.5, 0.95, 0.99), UInt64), + FirstSeen SimpleAggregateFunction(min, DateTime), + ApdexSatisfiedCount SimpleAggregateFunction(sum, UInt64), + ApdexToleratingCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Minute) +ORDER BY (OrgId, ServiceName, Minute, DeploymentEnv, ServiceNamespace, CommitSha) +TTL toDate(Minute) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS service_overview_spans ( + OrgId LowCardinality(String), + Timestamp DateTime, + ServiceName LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + TraceState String, + DeploymentEnv LowCardinality(String), + CommitSha LowCardinality(String), + SampleRate Float64 DEFAULT 1, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, Timestamp) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS service_platforms_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + DeploymentEnv LowCardinality(String), + K8sCluster SimpleAggregateFunction(max, String), + K8sPodName SimpleAggregateFunction(max, String), + K8sDeploymentName SimpleAggregateFunction(max, String), + K8sStatefulSetName SimpleAggregateFunction(max, String), + K8sDaemonSetName SimpleAggregateFunction(max, String), + K8sNamespaceName SimpleAggregateFunction(max, String), + CloudPlatform SimpleAggregateFunction(max, String), + CloudProvider SimpleAggregateFunction(max, String), + FaasName SimpleAggregateFunction(max, String), + MapleSdkType SimpleAggregateFunction(max, String), + ProcessRuntimeName SimpleAggregateFunction(max, String), + SpanCount SimpleAggregateFunction(sum, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS service_usage ( + OrgId LowCardinality(String), + ServiceName LowCardinality(String), + Hour DateTime, + LogCount UInt64, + LogSizeBytes UInt64, + TraceCount UInt64, + TraceSizeBytes UInt64, + SumMetricCount UInt64, + SumMetricSizeBytes UInt64, + GaugeMetricCount UInt64, + GaugeMetricSizeBytes UInt64, + HistogramMetricCount UInt64, + HistogramMetricSizeBytes UInt64, + ExpHistogramMetricCount UInt64, + ExpHistogramMetricSizeBytes UInt64 +) +ENGINE = SummingMergeTree +ORDER BY (OrgId, ServiceName, Hour) +TTL Hour + INTERVAL 365 DAY; + +CREATE TABLE IF NOT EXISTS session_events ( + OrgId LowCardinality(String), + SessionId String, + Timestamp DateTime64(9), + Seq UInt32 DEFAULT 0, + Type LowCardinality(String), + Url String DEFAULT '', + TraceId String DEFAULT '', + Level LowCardinality(String) DEFAULT '', + Message String DEFAULT '', + TargetSelector String DEFAULT '', + TargetText String DEFAULT '', + NetMethod LowCardinality(String) DEFAULT '', + NetUrl String DEFAULT '', + NetStatus UInt16 DEFAULT 0, + NetDurationMs UInt32 DEFAULT 0, + ErrorStack String DEFAULT '', + Attributes Map(String, String), + VisitorId String DEFAULT '', + UserId String DEFAULT '', + GroupId String DEFAULT '', + INDEX idx_type Type TYPE set(16) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, Timestamp, Seq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replay_events ( + OrgId LowCardinality(String), + SessionId String, + ChunkSeq UInt32, + Timestamp DateTime64(9), + DurationMs UInt32 DEFAULT 0, + EventCount UInt32 DEFAULT 0, + ByteSize UInt32 DEFAULT 0, + Events String, + IsCheckpoint UInt8 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, SessionId, ChunkSeq) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS session_replays ( + OrgId LowCardinality(String), + SessionId String, + StartTime DateTime64(9), + EndTime Nullable(DateTime64(9)), + DurationMs Nullable(UInt32), + Status LowCardinality(String), + UserId String, + UrlInitial String, + UserAgent String, + BrowserName LowCardinality(String), + OsName LowCardinality(String), + DeviceType LowCardinality(String), + Country LowCardinality(String) DEFAULT '', + ServiceName LowCardinality(String), + PageViews UInt32 DEFAULT 0, + ClickCount UInt32 DEFAULT 0, + ErrorCount UInt32 DEFAULT 0, + TraceIds Array(String) DEFAULT [], + ResourceAttributes Map(LowCardinality(String), String), + Version UInt32, + VisitorId String DEFAULT '', + VisitorIsNew UInt8 DEFAULT 0, + UserEmail String DEFAULT '', + UserName String DEFAULT '', + GroupId String DEFAULT '', + GroupName String DEFAULT '', + UserTraits Map(String, String) DEFAULT map(), + Referrer String DEFAULT '', + ReferrerHost LowCardinality(String) DEFAULT '', + UtmSource LowCardinality(String) DEFAULT '', + UtmMedium LowCardinality(String) DEFAULT '', + UtmCampaign LowCardinality(String) DEFAULT '', + UtmTerm String DEFAULT '', + UtmContent String DEFAULT '', + Host LowCardinality(String) DEFAULT '', + EntryPath String DEFAULT '', + ExitPath String DEFAULT '', + Language LowCardinality(String) DEFAULT '', + LastActivityAt Nullable(DateTime64(9)) +) +ENGINE = ReplacingMergeTree +PARTITION BY toDate(StartTime) +ORDER BY (OrgId, SessionId) +TTL toDate(StartTime) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS span_metrics_calls_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + MetricName LowCardinality(String), + SpanKind LowCardinality(String), + AttrFingerprint UInt64, + ResourceFingerprint UInt64, + StartTimeUnix DateTime64(9), + LastValue AggregateFunction(argMax, Float64, DateTime64(9)) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix) +TTL toDate(Hour) + INTERVAL 90 DAY; + +CREATE TABLE IF NOT EXISTS trace_detail_spans ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + ResourceAttributes Map(LowCardinality(String), String) +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, TraceId, SpanId) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS trace_list_mv ( + OrgId LowCardinality(String), + TraceId String, + Timestamp DateTime, + ServiceName LowCardinality(String), + SpanName String, + SpanKind LowCardinality(String), + Duration UInt64, + StatusCode LowCardinality(String), + HttpMethod LowCardinality(String), + HttpRoute String, + HttpStatusCode LowCardinality(String), + DeploymentEnv LowCardinality(String), + HasError UInt8, + TraceState String, + ServiceNamespace LowCardinality(String), + INDEX idx_service_namespace ServiceNamespace TYPE set(1000) GRANULARITY 4 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, Timestamp, TraceId) +TTL Timestamp + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces ( + OrgId LowCardinality(String), + Timestamp DateTime64(9), + TraceId String, + SpanId String, + ParentSpanId String, + TraceState String, + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + ServiceName LowCardinality(String), + ResourceSchemaUrl String, + ResourceAttributes Map(LowCardinality(String), String), + ScopeSchemaUrl String, + ScopeName String, + ScopeVersion String, + ScopeAttributes Map(LowCardinality(String), String), + Duration UInt64 DEFAULT 0, + StatusCode LowCardinality(String), + StatusMessage String, + SpanAttributes Map(LowCardinality(String), String), + EventsTimestamp Array(DateTime64(9)), + EventsName Array(LowCardinality(String)), + EventsAttributes Array(Map(LowCardinality(String), String)), + LinksTraceId Array(String), + LinksSpanId Array(String), + LinksTraceState Array(String), + LinksAttributes Array(Map(LowCardinality(String), String)), + SampleRate Float64 DEFAULT multiIf(SpanAttributes['SampleRate'] != '' AND toFloat64OrZero(SpanAttributes['SampleRate']) >= 1.0, toFloat64OrZero(SpanAttributes['SampleRate']), match(TraceState, 'th:[0-9a-f]+'), 1.0 / greatest(1.0 - reinterpretAsUInt64(reverse(unhex(rightPad(extract(TraceState, 'th:([0-9a-f]+)'), 16, '0')))) / pow(2.0, 64), 0.0001), 1.0), + IsEntryPoint UInt8 DEFAULT if(SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '', 1, 0), + ResourceAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ResourceAttributes), mapValues(ResourceAttributes)), + ScopeAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(ScopeAttributes), mapValues(ScopeAttributes)), + SpanAttributeItems Array(String) DEFAULT arrayMap((k, v) -> concat(k, char(31), v), mapKeys(SpanAttributes), mapValues(SpanAttributes)), + INDEX idx_trace_id TraceId TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_keys mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_span_attr_vals mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_keys mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_resource_attr_vals mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_keys mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1, + INDEX idx_scope_attr_vals mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1 +) +ENGINE = MergeTree +PARTITION BY toDate(Timestamp) +ORDER BY (OrgId, ServiceName, SpanName, toDateTime(Timestamp)) +TTL toDate(Timestamp) + INTERVAL 30 DAY; + +CREATE TABLE IF NOT EXISTS traces_aggregates_hourly ( + OrgId LowCardinality(String), + Hour DateTime, + ServiceName LowCardinality(String), + SpanName LowCardinality(String), + SpanKind LowCardinality(String), + StatusCode LowCardinality(String), + IsEntryPoint UInt8, + DeploymentEnv LowCardinality(String), + WeightedCount SimpleAggregateFunction(sum, Float64), + WeightedDurationSum SimpleAggregateFunction(sum, Float64), + WeightedErrorCount SimpleAggregateFunction(sum, Float64), + DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), UInt64, UInt32), + DurationMin SimpleAggregateFunction(min, UInt64), + DurationMax SimpleAggregateFunction(max, UInt64) +) +ENGINE = AggregatingMergeTree +PARTITION BY toDate(Hour) +ORDER BY (OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv) +TTL toDate(Hour) + INTERVAL 365 DAY; + +CREATE MATERIALIZED VIEW IF NOT EXISTS ai_trace_index_mv TO ai_trace_index AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanAttributes['maple_ai.session.id'] AS SessionId, + SpanAttributes['maple_ai.vendor.id'] AS VendorId, + ServiceName + FROM traces + WHERE SpanAttributes['maple_ai.vendor.id'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_by_time_mv TO error_events_by_time AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_events_mv TO error_events AS +WITH + arrayFirstIndex(n -> n = 'exception', EventsName) AS _ei, + if(_ei > 0, EventsAttributes[_ei]['exception.type'], '') AS _exType, + if(_ei > 0, EventsAttributes[_ei]['exception.message'], StatusMessage) AS _exMsg, + if(_ei > 0, EventsAttributes[_ei]['exception.stacktrace'], '') AS _exStack, + -- Frame lines are matched by SHAPE, not by "contains :NUMBER". The old + -- rule accepted any line with a colon-digit, which let non-frame lines + -- in: Drizzle's `params: ` line, and the `Type: message` + -- header (`Code: 62`, `position 1628`, embedded timestamps). Row values + -- and message text then entered the hash and split one bug into + -- thousands of issues — 23,035 fingerprints for six real + -- AnomalyPersistenceError call sites, 15,051 for thirteen DatabaseError + -- ones. + -- + -- The pattern is rendered from FRAME_LINE_PATTERN in fingerprint.ts, + -- as is every redaction below. They used to be hand-copied here, which + -- let the reference implementation the tests exercise drift away from + -- the SQL that actually runs, silently. + arraySlice( + arrayFilter( + line -> match(line, '^[ \\t]*at |^[ \\t]*File "|^[ \\t]+from [^ ]+:[0-9]+|^[^ \\t@]+@[^ \\t]*:[0-9]+|^[ \\t]+[^ \\t]+\\.(go|rs):[0-9]+|^[0-9]+ +\\S.* +0x[0-9a-fA-F]+'), + splitByChar('\n', _exStack) + ), + 1, 3 + ) AS _rawFrames, + -- Redact every volatile token a frame line can carry: the URL origin + -- (so preview hosts share one fingerprint), Vite's 8-char bundle + -- content hash (so a deploy does not re-split every triaged browser and + -- Worker issue), then line numbers, hex pointers and long id runs. See + -- FRAME_REDACTIONS for the order and the reasoning. + arrayMap( + line -> replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(line, 'https?://[^/ )]+', ''), '-[A-Za-z0-9_-]{8}\\.js', '.js'), '-[A-Za-z0-9_-]{8}\\.css', '.css'), ':[0-9]+|line [0-9]+|0x[0-9a-fA-F]+|[0-9a-fA-F]{8,}|[0-9]{6,}', ''), + _rawFrames + ) AS _topFrames, + if(length(_topFrames) > 0, _topFrames[1], '') AS _topFrame, + arrayStringConcat(_topFrames, '\n') AS _fpFrames, + -- JSON detection for the message signature below. + isValidJSON(StatusMessage) AS _isJson, + _isJson AND JSONType(StatusMessage) = 'Object' AS _isJsonObj, + -- General, KEY-NAME-AGNOSTIC canonical signature: iterate ALL top-level + -- keys, redact volatile tokens (long hex / numbers) in each raw value, then + -- sort by "key=value" so key order & whitespace don't matter. No assumption + -- about which keys exist — works for any producer's JSON shape. (Nested + -- objects are hashed as their raw substring; only top-level is canonicalized.) + arrayStringConcat( + arraySort( + arrayMap( + kv -> concat(kv.1, '=', replaceRegexpAll(kv.2, '[0-9a-fA-F]{8,}|[0-9]+', '#')), + JSONExtractKeysAndValuesRaw(StatusMessage) + ) + ), + '|' + ) AS _jsonSig, + -- The message signature is folded in ALWAYS, not only when there are no + -- frames. Bundled runtimes minify every module into one file, so the top + -- three frames of a Worker error are `toDatabaseError (worker.js)` for + -- every failing query alike: on frames alone, 25 distinct DatabaseError + -- bugs (316k occurrences) collapse into a single issue. The signature + -- restores that discrimination, and it cannot reinflate cardinality the + -- way a raw prefix would because everything variable is redacted first: + -- emails, URL origins, home directories, query strings, quoted values, + -- then ids and every digit run. See MSG_TEXT_REDACTIONS for the order, + -- what is deliberately kept, and the one residual it cannot reach. + multiIf( + _isJsonObj, _jsonSig, + substringUTF8( + replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(substringUTF8(StatusMessage, 1, 400), '[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}', 'EMAIL'), 'https?://[^/ )"]+', ''), '/(Users|home)/[^/ ]+', '/~'), '[?][A-Za-z0-9_]+=[^ )"]*', '?#'), '\'[^\' ]*/[^\' ]*\'|\'[^\' ]{25,}\'', '\'#\''), '"[^" ]*/[^" ]*"|"[^" ]{25,}"', '"#"'), '`[^` ]*/[^` ]*`|`[^` ]{25,}`', '`#`'), '[0-9a-fA-F-]{6,}|[0-9]+', '#'), + 1, 120 + ) + ) AS _msgSig, + -- Display-only, best-effort human label (decoupled from the fingerprint: + -- many labels may map to one hash). The broad key list here is a DISPLAY + -- heuristic only; the fingerprint above makes no key-name assumption. + multiIf( + JSONExtractString(StatusMessage, 'title') != '', JSONExtractString(StatusMessage, 'title'), + JSONExtractString(StatusMessage, 'message') != '', JSONExtractString(StatusMessage, 'message'), + JSONExtractString(StatusMessage, 'error') != '', JSONExtractString(StatusMessage, 'error'), + JSONExtractString(StatusMessage, '_tag') != '', JSONExtractString(StatusMessage, '_tag'), + JSONExtractString(StatusMessage, 'reason') != '', JSONExtractString(StatusMessage, 'reason'), + JSONExtractString(StatusMessage, 'name') != '', JSONExtractString(StatusMessage, 'name'), + JSONExtractString(StatusMessage, 'type') != '', extract(JSONExtractString(StatusMessage, 'type'), '([^/]+)$'), + 'JSON error' + ) AS _jsonLabel, + multiIf( + StatusMessage = '', 'Unknown Error', + position(StatusMessage, '{ readonly') = 1 OR position(StatusMessage, '└─') > 0, + if( + extract(StatusMessage, 'readonly (\\w+)') != '', + concat('Schema parse error: ', extract(StatusMessage, 'readonly (\\w+)')), + 'Schema parse error' + ), + _isJsonObj OR position(StatusMessage, '[') = 1, _jsonLabel, + left(StatusMessage, multiIf( + position(StatusMessage, ': ') > 3, toInt64(position(StatusMessage, ': ')) - 1, + position(StatusMessage, ' (') > 3, toInt64(position(StatusMessage, ' (')) - 1, + position(StatusMessage, '\n') > 3, toInt64(position(StatusMessage, '\n')) - 1, + least(toInt64(length(StatusMessage)), 150) + )) + ) AS _statusLabel, + if(_exType != '', _exType, _statusLabel) AS _errorLabel, + -- Both semconv spellings; the current key wins when both are present. + toUInt16OrZero( + if( + SpanAttributes['http.response.status_code'] != '', + SpanAttributes['http.response.status_code'], + SpanAttributes['http.status_code'] + ) + ) AS _httpStatus + SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + _exType AS ExceptionType, + _exMsg AS ExceptionMessage, + _exStack AS ExceptionStacktrace, + _topFrame AS TopFrame, + cityHash64(OrgId, ServiceName, _exType, _fpFrames, _msgSig) AS FingerprintHash, + StatusMessage, + Duration, + _errorLabel AS ErrorLabel, + ResourceAttributes['service.version'] AS ServiceVersion + FROM traces + WHERE StatusCode = 'Error' + -- Client-side runtimes (notably the native Cloudflare Workers + -- observability) mark ANY non-2xx fetch span as Error, so 404s from bot + -- traffic arrived here as unlabelled "Unknown Error" issues. Drop a + -- span only when all three hold: 4xx, no exception event, and no + -- exception type. 5xx and anything carrying an exception still count, + -- and SpanKind is deliberately not consulted — these are Client spans. + AND NOT ( + _httpStatus >= 400 AND _httpStatus < 500 + AND _ei = 0 + AND _exType = '' + ); + +CREATE MATERIALIZED VIEW IF NOT EXISTS error_fingerprints_minutely_mv TO error_fingerprints_minutely AS +SELECT + OrgId, + toStartOfMinute(Timestamp) AS Minute, + FingerprintHash, + anyLast(ServiceName) AS ServiceName, + anyLast(ExceptionType) AS ExceptionType, + anyLast(ExceptionMessage) AS ExceptionMessage, + anyLast(ErrorLabel) AS ErrorLabel, + anyLast(TopFrame) AS TopFrame, + count() AS OccurrenceCount, + min(Timestamp) AS FirstSeen, + max(Timestamp) AS LastSeen, + -- Distinct builds, not a sample: see ServiceVersions on the datasource. + groupUniqArray(ServiceVersion) AS ServiceVersions + FROM error_events + GROUP BY OrgId, Minute, FingerprintHash; + +CREATE MATERIALIZED VIEW IF NOT EXISTS identity_links_mv TO identity_links AS +SELECT + OrgId, + VisitorId, + UserId, + StartTime AS FirstSeen + FROM session_replays + WHERE VisitorId != '' AND UserId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(LogAttributes)) AS AttributeKey, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + WHERE LogAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS log_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'log' AS AttributeScope, + count() AS UsageCount + FROM logs + ARRAY JOIN + mapKeys(LogAttributes) AS AttributeKey, + mapValues(LogAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS logs_aggregates_hourly_mv TO logs_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(TimestampTime) AS Hour, + ServiceName, + SeverityText, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS Count, + sum(length(Body) + 200) AS SizeBytes, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM logs + GROUP BY OrgId, Hour, ServiceName, SeverityText, DeploymentEnv, ServiceNamespace; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + arrayJoin(mapKeys(Attributes)) AS AttributeKey, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + WHERE Attributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + AttributeKey, + AttributeValue, + 'metric' AS AttributeScope, + count() AS UsageCount + FROM metrics_sum + ARRAY JOIN + mapKeys(Attributes) AS AttributeKey, + mapValues(Attributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_exp_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'exponential_histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_exponential_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'gauge' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_gauge + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'histogram' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + toUInt8(0) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_histogram + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 'sum' AS MetricType, + ServiceName, + MetricName, + anyLast(MetricDescription) AS MetricDescription, + anyLast(MetricUnit) AS MetricUnit, + anyLast(toUInt8(IsMonotonic)) AS IsMonotonic, + count() AS DataPointCount, + min(toDateTime(TimeUnix)) AS FirstSeen, + max(toDateTime(TimeUnix)) AS LastSeen + FROM metrics_sum + GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId + FROM session_events + WHERE Type IN ('navigation', 'custom'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging', + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc', + 'http' + ) AS TargetType, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'], + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'], + '' + ) AS TargetSystem, + multiIf( + coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', + if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']), + SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', + if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']), + if(SpanAttributes['server.address'] != '', + SpanAttributes['server.address'], + if(SpanAttributes['http.host'] != '', + SpanAttributes['http.host'], + SpanAttributes['url.authority'])) + ) AS TargetName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND SpanAttributes['db.system.name'] = '' + AND ServiceName != '' + AND ( + SpanAttributes['server.address'] != '' + OR SpanAttributes['http.host'] != '' + OR SpanAttributes['url.authority'] != '' + OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' + OR SpanAttributes['messaging.system'] != '' + OR SpanAttributes['rpc.service'] != '' + OR SpanAttributes['rpc.system'] != '' + ) + GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv + HAVING TargetName != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') + AND ParentSpanId != ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(Duration / 1000000) AS DurationSumMs, + max(Duration / 1000000) AS MaxDurationMs, + countIf(TraceState LIKE '%th:%') AS SampledSpanCount, + countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount, + sum(SampleRate) AS SampleRateSum, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_query_shapes_hourly_mv TO service_map_db_query_shapes_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem, + if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + coalesce( + nullIf(SpanAttributes['db.query.fingerprint'], ''), + nullIf(SpanAttributes['db.statement.fingerprint'], ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', toString(cityHash64(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(replaceRegexpAll(lower(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement'])), '\'[^\']*\'', '?'), '\\bin\\s*\\([^)]*\\)', 'in (?)'), '[0-9]+(\\.[0-9]+)?', '?'), '\\s+', ' '), '^\\s+|\\s+$', ''))), ''), ''), + toString(cityHash64(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +))) +) AS QueryKey, + any(substring(coalesce( + nullIf(SpanAttributes['db.query.summary'], ''), + nullIf(if(SpanAttributes['db.operation.name'] != '', trimBoth(concat(SpanAttributes['db.operation.name'], if(coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace']) != '', concat(' ', coalesce(nullIf(SpanAttributes['db.collection.name'], ''), SpanAttributes['db.namespace'])), ''))), ''), ''), + nullIf(if(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']) != '', trimBoth(concat(upper(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '^\\s*(\\w+)')), if(extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)') != '', concat(' ', extract(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), '(?i)(?:from|into|update|join|table)\\s+\\W?([\\w.]+)')), ''))), ''), ''), + nullIf(SpanAttributes['query.context'], ''), + nullIf(SpanAttributes['db.operation.name'], ''), + nullIf(SpanAttributes['db.operation'], ''), + SpanName +), 1, 220)) AS QueryLabel, + any(substring(coalesce(nullIf(SpanAttributes['db.query.text'], ''), SpanAttributes['db.statement']), 1, 1000)) AS SampleStatement, + count() AS CallCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sum(SampleRate) AS EstimatedCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration) * SampleRate / 1000000) AS WeightedDurationSumMs, + quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles + FROM traces + WHERE SpanKind IN ('Client', 'Producer') + AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != '' + AND ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv, QueryKey; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_edges_hourly_ingest_mv TO service_map_edges_hourly AS +SELECT + OrgId, + Hour, + SourceService, + TargetService, + DeploymentEnv, + CallCount, + ErrorCount, + DurationSumMs, + MaxDurationMs, + SampledSpanCount, + UnsampledSpanCount, + SampleRateSum + FROM service_map_edges_hourly_ingest; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_spans_mv TO service_map_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + TraceId, + SpanId, + ParentSpanId, + ServiceName, + SpanKind, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv + FROM traces + WHERE SpanKind IN ('Client', 'Producer', 'Server', 'Consumer'); + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_hourly_mv TO service_operations_hourly AS +SELECT + OrgId, + toStartOfHour(Minute) AS Hour, + ServiceName, + DeploymentEnv, + SpanName, + sum(SpanCount) AS SpanCount, + sum(EstimatedSpanCount) AS EstimatedSpanCount, + sum(ErrorCount) AS ErrorCount, + sum(EstimatedErrorCount) AS EstimatedErrorCount, + sum(DurationSum) AS DurationSum, + quantilesTDigestMergeState(0.5, 0.95)(DurationQuantiles) AS DurationQuantiles, + sum(ClassifiedSpanCount) AS ClassifiedSpanCount, + sum(ServerSpanCount) AS ServerSpanCount, + sum(RoutedSpanCount) AS RoutedSpanCount + FROM service_operations_minutely + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_operations_minutely_mv TO service_operations_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + if(((SpanName LIKE 'http.server %' OR SpanName IN ('GET', 'POST', 'PUT', 'PATCH', 'DELETE', 'HEAD', 'OPTIONS')) AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != '')), concat(if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), ' ', if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path'])), SpanName) AS SpanName, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95)(Duration) AS DurationQuantiles, + count() AS ClassifiedSpanCount, + countIf(SpanKind IN ('Server', 'Consumer')) AS ServerSpanCount, + countIf(SpanAttributes['http.route'] != '') AS RoutedSpanCount + FROM traces + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, SpanName; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_hourly_mv TO service_overview_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_minutely_mv TO service_overview_minutely AS +SELECT + OrgId, + toStartOfMinute(toDateTime(Timestamp)) AS Minute, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['service.namespace'] AS ServiceNamespace, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + count() AS SpanCount, + sum(SampleRate) AS EstimatedSpanCount, + countIf(StatusCode = 'Error') AS ErrorCount, + sumIf(SampleRate, StatusCode = 'Error') AS EstimatedErrorCount, + sum(toFloat64(Duration)) AS DurationSum, + quantilesTDigestState(0.5, 0.95, 0.99)(Duration) AS DurationQuantiles, + min(toDateTime(Timestamp)) AS FirstSeen, + countIf(StatusCode != 'Error' AND Duration < 500000000) AS ApdexSatisfiedCount, + countIf(StatusCode != 'Error' AND Duration >= 500000000 AND Duration < 2000000000) AS ApdexToleratingCount + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = '' + GROUP BY OrgId, Minute, ServiceName, DeploymentEnv, ServiceNamespace, CommitSha; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_overview_spans_mv TO service_overview_spans AS +SELECT + OrgId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + Duration, + StatusCode, + TraceState, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + ResourceAttributes['vcs.ref.head.revision'] AS CommitSha, + SampleRate, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE SpanKind IN ('Server', 'Consumer') OR ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_platforms_hourly_mv TO service_platforms_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + max(ResourceAttributes['k8s.cluster.name']) AS K8sCluster, + max(ResourceAttributes['k8s.pod.name']) AS K8sPodName, + max(ResourceAttributes['k8s.deployment.name']) AS K8sDeploymentName, + max(ResourceAttributes['k8s.statefulset.name']) AS K8sStatefulSetName, + max(ResourceAttributes['k8s.daemonset.name']) AS K8sDaemonSetName, + max(ResourceAttributes['k8s.namespace.name']) AS K8sNamespaceName, + max(ResourceAttributes['cloud.platform']) AS CloudPlatform, + max(ResourceAttributes['cloud.provider']) AS CloudProvider, + max(ResourceAttributes['faas.name']) AS FaasName, + max(ResourceAttributes['maple.sdk.type']) AS MapleSdkType, + max(ResourceAttributes['process.runtime.name']) AS ProcessRuntimeName, + count() AS SpanCount + FROM traces + WHERE ServiceName != '' + GROUP BY OrgId, Hour, ServiceName, DeploymentEnv; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_logs_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(TimestampTime) AS Hour, + count() AS LogCount, + sum(length(Body) + 200) AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM logs + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_exp_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + count() AS ExpHistogramMetricCount, + count() * 300 AS ExpHistogramMetricSizeBytes + FROM metrics_exponential_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_gauge_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + count() AS GaugeMetricCount, + count() * 150 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_gauge + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_histogram_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + count() AS HistogramMetricCount, + count() * 250 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_histogram + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_metrics_sum_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + 0 AS TraceCount, + 0 AS TraceSizeBytes, + count() AS SumMetricCount, + count() * 150 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM metrics_sum + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS service_usage_traces_mv TO service_usage AS +SELECT + OrgId, + ServiceName, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + 0 AS LogCount, + 0 AS LogSizeBytes, + count() AS TraceCount, + sum(length(SpanName) + 300) AS TraceSizeBytes, + 0 AS SumMetricCount, + 0 AS SumMetricSizeBytes, + 0 AS GaugeMetricCount, + 0 AS GaugeMetricSizeBytes, + 0 AS HistogramMetricCount, + 0 AS HistogramMetricSizeBytes, + 0 AS ExpHistogramMetricCount, + 0 AS ExpHistogramMetricSizeBytes + FROM traces + GROUP BY OrgId, ServiceName, Hour; + +CREATE MATERIALIZED VIEW IF NOT EXISTS span_metrics_calls_hourly_mv TO span_metrics_calls_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(TimeUnix)) AS Hour, + ServiceName, + MetricName, + Attributes['span.kind'] AS SpanKind, + cityHash64(mapKeys(Attributes), mapValues(Attributes)) AS AttrFingerprint, + cityHash64(mapKeys(ResourceAttributes), mapValues(ResourceAttributes)) AS ResourceFingerprint, + StartTimeUnix, + argMaxState(Value, TimeUnix) AS LastValue + FROM metrics_sum + -- 'traces.span.metrics.calls' is the name the collector actually emits: + -- spanmetricsconnector output is namespaced by the pipeline it is attached + -- to. Without it this MV matched nothing and the target sat at 0 rows since + -- it was created, while ~880k rows / 2 days of the real counter flowed past + -- into metrics_sum and every read fell back to the raw window-function scan + -- (~7s p95 -- see queries/metrics.ts). Keep this list in sync with + -- SPAN_METRICS_CALLS_NAMES on the read side. + WHERE MetricName IN ('span.metrics.calls', 'calls', 'traces.span.metrics.calls') AND IsMonotonic + GROUP BY OrgId, Hour, ServiceName, MetricName, SpanKind, AttrFingerprint, ResourceFingerprint, StartTimeUnix; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_detail_spans_mv TO trace_detail_spans AS +SELECT + OrgId, + Timestamp, + TraceId, + SpanId, + ParentSpanId, + SpanName, + SpanKind, + ServiceName, + Duration, + StatusCode, + StatusMessage, + SpanAttributes, + ResourceAttributes + FROM traces; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_list_mv_mv TO trace_list_mv AS +SELECT + OrgId, + TraceId, + toDateTime(Timestamp) AS Timestamp, + ServiceName, + if( + (SpanName LIKE 'http.server %' OR SpanName IN ('GET','POST','PUT','PATCH','DELETE','HEAD','OPTIONS')) + AND (SpanAttributes['http.route'] != '' OR SpanAttributes['url.path'] != ''), + concat( + if(SpanName LIKE 'http.server %', replaceOne(SpanName, 'http.server ', ''), SpanName), + ' ', + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], SpanAttributes['url.path']) + ), + SpanName + ) AS SpanName, + SpanKind, + Duration, + StatusCode, + if(SpanAttributes['http.method'] != '', SpanAttributes['http.method'], SpanAttributes['http.request.method']) AS HttpMethod, + if(SpanAttributes['http.route'] != '', SpanAttributes['http.route'], if(SpanAttributes['url.path'] != '', SpanAttributes['url.path'], SpanAttributes['http.target'])) AS HttpRoute, + if(SpanAttributes['http.status_code'] != '', SpanAttributes['http.status_code'], SpanAttributes['http.response.status_code']) AS HttpStatusCode, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + toUInt8( + StatusCode = 'Error' + OR (SpanAttributes['http.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.status_code']) >= 500) + OR (SpanAttributes['http.response.status_code'] != '' AND toUInt16OrZero(SpanAttributes['http.response.status_code']) >= 500) + ) AS HasError, + TraceState, + ResourceAttributes['service.namespace'] AS ServiceNamespace + FROM traces + WHERE ParentSpanId = ''; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(ResourceAttributes)) AS AttributeKey, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE ResourceAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_resource_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'resource' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(ResourceAttributes) AS AttributeKey, + mapValues(ResourceAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_keys_mv TO attribute_keys_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + arrayJoin(mapKeys(SpanAttributes)) AS AttributeKey, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + WHERE SpanAttributes != map() + GROUP BY OrgId, Hour, AttributeKey, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS trace_span_attribute_values_mv TO attribute_values_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + AttributeKey, + AttributeValue, + 'span' AS AttributeScope, + count() AS UsageCount + FROM traces + ARRAY JOIN + mapKeys(SpanAttributes) AS AttributeKey, + mapValues(SpanAttributes) AS AttributeValue + WHERE AttributeValue != '' + AND length(AttributeValue) <= 128 + AND NOT (length(AttributeValue) > 4 AND match(AttributeValue, '^[0-9]+([.][0-9]+)?$')) + AND NOT match(AttributeKey, '(_id|[.]id|Id|_ns)$') + AND AttributeKey NOT LIKE 'http.request.header.%' + AND AttributeKey NOT LIKE 'http.response.header.%' + GROUP BY OrgId, Hour, AttributeKey, AttributeValue, AttributeScope; + +CREATE MATERIALIZED VIEW IF NOT EXISTS traces_aggregates_hourly_mv TO traces_aggregates_hourly AS +SELECT + OrgId, + toStartOfHour(toDateTime(Timestamp)) AS Hour, + ServiceName, + SpanName, + SpanKind, + StatusCode, + IsEntryPoint, + coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv, + sum(SampleRate) AS WeightedCount, + sum(toFloat64(Duration) * SampleRate) AS WeightedDurationSum, + sumIf(SampleRate, StatusCode = 'Error') AS WeightedErrorCount, + quantilesTDigestWeightedState(0.5, 0.95, 0.99)(Duration, toUInt32(SampleRate)) AS DurationQuantiles, + min(Duration) AS DurationMin, + max(Duration) AS DurationMax + FROM traces + GROUP BY OrgId, Hour, ServiceName, SpanName, SpanKind, StatusCode, IsEntryPoint, DeploymentEnv; diff --git a/apps/cli/src/server/schema/local-schema.sql b/apps/cli/src/server/schema/local-schema.sql index 5390dc109..0bb35482e 100644 --- a/apps/cli/src/server/schema/local-schema.sql +++ b/apps/cli/src/server/schema/local-schema.sql @@ -1,7 +1,7 @@ -- This file is generated by scripts/generate-clickhouse-schema-sql.ts -- Do not edit manually. Run `bun run clickhouse:schema` to regenerate. --- projectRevision: bf3419c18e581ffab5fa4e24aa56f421ac9c3d5a1ce734596747f2331d7d2ae0 --- localSchemaVersion: 15 +-- projectRevision: 4545ddafc64d7bd0234a65c2442cb615373026c6fcf1d2f4b3f0f0e6b5ca88d9 +-- localSchemaVersion: 16 CREATE TABLE IF NOT EXISTS ai_trace_index ( OrgId LowCardinality(String), @@ -359,8 +359,11 @@ CREATE TABLE IF NOT EXISTS product_events ( Url String DEFAULT '', ServiceName LowCardinality(String) DEFAULT '', Attributes Map(String, String) DEFAULT map(), + TraceId String DEFAULT '', + SpanId String DEFAULT '', INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4, - INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4 + INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4, + INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4 ) ENGINE = MergeTree PARTITION BY toDate(Timestamp) @@ -1349,10 +1352,56 @@ SELECT path(Url) AS PagePath, Url, '' AS ServiceName, - Attributes + Attributes, + '' AS TraceId, + '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom'); +CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId + FROM traces + WHERE SpanAttributes['maple.product_event.name'] != ''; + CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS SELECT OrgId, diff --git a/apps/cli/test/local-store-migrations.test.ts b/apps/cli/test/local-store-migrations.test.ts index e6c864229..bf8112d05 100644 --- a/apps/cli/test/local-store-migrations.test.ts +++ b/apps/cli/test/local-store-migrations.test.ts @@ -30,6 +30,7 @@ import { LOCAL_SCHEMA_V13_MANIFEST, LOCAL_SCHEMA_V14, LOCAL_SCHEMA_V15, + LOCAL_SCHEMA_V16, SCHEMA_DIGEST, SCHEMA_FINGERPRINT, } from "../src/server/schema-identity" @@ -77,16 +78,16 @@ import { tmpdir } from "node:os" import { join } from "node:path" describe("current local schema identity", () => { - it("matches the generated v15 revision and keeps the issue-297 identity frozen", () => { - expect(SCHEMA_FINGERPRINT).toBe("24710426938d7b4a") - expect(SCHEMA_DIGEST).toBe("24710426938d7b4adf615f87f78315c2a5c6145a0029c4f244a339888b25f6d3") + it("matches the generated v16 revision and keeps the issue-297 identity frozen", () => { + expect(SCHEMA_FINGERPRINT).toBe("3cfe5f649a11853a") + expect(SCHEMA_DIGEST).toBe("3cfe5f649a11853ae87021dfb611b0ef76cbd6be4a597d247fb2dff4e717c59c") expect(ISSUE_297_TARGET_SCHEMA_PROJECT_REVISION).toBe( "506bc745f7a7eca202ec905a6403a6815e86413faf0cd3cbbf73881023edce91", ) expect(CURRENT_SCHEMA_PROJECT_REVISION).toMatch(/^[0-9a-f]{64}$/) expect(LOCAL_SCHEMA_MANIFEST.objects.length).toBeGreaterThan(60) - expect(CURRENT_LOCAL_SCHEMA.version).toBe(15) - expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V15) + expect(CURRENT_LOCAL_SCHEMA.version).toBe(16) + expect(CURRENT_LOCAL_SCHEMA).toEqual(LOCAL_SCHEMA_V16) const logs = LOCAL_SCHEMA_MANIFEST.objects.find((object) => object.name === "logs") expect(logs?.columns.some((column) => column.name.startsWith("idx_"))).toBe(false) expect(logs?.indexes).toContain("idx_lower_body") @@ -170,6 +171,7 @@ describe("current local schema identity", () => { "identity_links_mv", "product_events", "product_events_mv", + "product_events_traces_mv", ]) const errorEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "error_events_mv", @@ -196,7 +198,7 @@ describe("current local schema identity", () => { expect(productEvents?.engine).toBe("MergeTree") expect(productEvents?.orderBy).toBe("(OrgId, Timestamp, VisitorId, SessionId, Seq)") expect(productEvents?.ttl).toContain("365 DAY") - expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id"]) + expect(productEvents?.indexes).toEqual(["idx_event_name", "idx_user_id", "idx_trace_id"]) expect(productEvents?.columns.map((column) => column.name)).toEqual([ "OrgId", "Timestamp", @@ -213,6 +215,10 @@ describe("current local schema identity", () => { "Url", "ServiceName", "Attributes", + // Appended, not inserted: `ALTER TABLE … ADD COLUMN` puts them last, and + // every projection into this table has to match that order. + "TraceId", + "SpanId", ]) const productEventsView = LOCAL_SCHEMA_MANIFEST.objects.find( (object) => object.name === "product_events_mv", @@ -264,6 +270,7 @@ describe("current local schema identity", () => { expect([...currentSchemaNames].filter((name) => !v13Names.has(name))).toEqual([ "ai_trace_index", "ai_trace_index_mv", + "product_events_traces_mv", ]) expect([...v13Names].filter((name) => !currentSchemaNames.has(name))).toEqual([]) const aiTraceIndex = LOCAL_SCHEMA_MANIFEST.objects.find( @@ -312,6 +319,7 @@ describe("local migration registry", () => { "local-0012-to-0013-service-operations-discriminators", "local-0013-to-0014-ai-trace-index", "local-0014-to-0015-commit-sha-vcs-revision", + "local-0015-to-0016-product-events-from-traces", ]) expect(chain[0]?.from.fingerprint).toBe(LEGACY_SCHEMA_FINGERPRINT) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V1) @@ -358,7 +366,7 @@ describe("local migration registry", () => { // One past the current tip — bump alongside LOCAL_SCHEMA_VERSION, or this // stops testing the future-store guard and starts testing the // unknown-fingerprint one. - { ...CURRENT_LOCAL_SCHEMA, version: 16, fingerprint: "future", digest: SCHEMA_DIGEST }, + { ...CURRENT_LOCAL_SCHEMA, version: 17, fingerprint: "future", digest: SCHEMA_DIGEST }, CURRENT_LOCAL_SCHEMA, ), ).toThrow(/newer than this build/) @@ -1362,6 +1370,7 @@ describe("v10 -> v11 product events module", () => { "local-0012-to-0013-service-operations-discriminators", "local-0013-to-0014-ai-trace-index", "local-0014-to-0015-commit-sha-vcs-revision", + "local-0015-to-0016-product-events-from-traces", ]) expect(chain[0]?.to).toEqual(LOCAL_SCHEMA_V11) // The dropped table is declared, and the backfilled ones say what they diff --git a/apps/cli/test/native-local-store-migration.sh b/apps/cli/test/native-local-store-migration.sh index 960c789e9..9ee23e691 100755 --- a/apps/cli/test/native-local-store-migration.sh +++ b/apps/cli/test/native-local-store-migration.sh @@ -142,7 +142,7 @@ grep -q "local store migrated" "$ROOT/migrate.out" || fail "native migration did # must be bumped in lockstep with LOCAL_SCHEMA_VERSION and the matching # LOCAL_SCHEMA_V.fingerprint in apps/cli/src/server/schema-identity.ts; # leaving it on the previous version is what makes this step fail after a bump. -jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 15 and .schema == "24710426938d7b4a"' \ +jq -e '.formatVersion == 2 and .activation == "active" and .schemaVersion == 16 and .schema == "3cfe5f649a11853a"' \ "$ROOT/maple-store-version.json" >/dev/null || fail "native migration wrote the wrong active identity" step "reopening promoted store in a fresh server" diff --git a/apps/ingest/src/clickhouse_insert_mappings.rs b/apps/ingest/src/clickhouse_insert_mappings.rs index 1cfe86ea0..3066bcda4 100644 --- a/apps/ingest/src/clickhouse_insert_mappings.rs +++ b/apps/ingest/src/clickhouse_insert_mappings.rs @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-insert-mappings.ts // Do not edit manually. -pub const PROJECT_REVISION: &str = "bf3419c18e581ffab5fa4e24aa56f421ac9c3d5a1ce734596747f2331d7d2ae0"; +pub const PROJECT_REVISION: &str = "4545ddafc64d7bd0234a65c2442cb615373026c6fcf1d2f4b3f0f0e6b5ca88d9"; // Gate for BYO-ClickHouse ingest readiness — the migration version, NOT the // Tinybird-coupled PROJECT_REVISION. Compared against // org_clickhouse_settings.schema_version. See @maple/domain/clickhouse diff --git a/apps/web/src/api/warehouse/product-events.ts b/apps/web/src/api/warehouse/product-events.ts index 687d2b718..79a93b770 100644 --- a/apps/web/src/api/warehouse/product-events.ts +++ b/apps/web/src/api/warehouse/product-events.ts @@ -5,10 +5,13 @@ // `keyBy`, the session step, and the breakdown grouping. import { Effect, Schema } from "effect" +import { TraceId } from "@maple/domain" import { ProductEventNamesRequest, + ProductEventsForTraceRequest, ProductEventsFunnelBreakdownRequest, ProductEventsFunnelRequest, + ProductEventTraceSamplesRequest, } from "@maple/domain/http" import { FUNNEL_WIDGET_BREAKDOWN_LIMIT, @@ -63,6 +66,103 @@ const getProductEventNamesEffect = Effect.fn("QueryEngine.getProductEventNames") return { data: result.data satisfies ReadonlyArray } }) +// Trace ↔ product event. +// +// A span annotated in the customer's own code with `maple.product_event.name` +// becomes a `product_events` row carrying its `TraceId`, and these two read that +// column from either end: the trace view lists what a request accomplished, and +// an event name lists the requests that accomplished it. + +// `TraceId`, not a plain string: `decodeInput` is the boundary that turns a +// malformed id in the URL into a decode failure here rather than a warehouse +// error four hops later. +const ProductEventsForTraceInputSchema = Schema.Struct({ + ...TimeWindowFields, + traceId: TraceId, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventsForTraceInput = (typeof ProductEventsForTraceInputSchema)["Encoded"] + +export interface TraceProductEvent { + timestamp: string + eventName: string + /** The annotated span within the trace — deep-links to it in the waterfall. */ + spanId: string + serviceName: string + userId: string + groupId: string + visitorId: string + sessionId: string + /** `maple.product_event.prop.*` attributes, prefix stripped. */ + attributes: Record +} + +export function getProductEventsForTrace({ data }: { data: GetProductEventsForTraceInput }) { + return getProductEventsForTraceEffect({ data }) +} + +const getProductEventsForTraceEffect = Effect.fn("QueryEngine.getProductEventsForTrace")(function* ({ + data, +}: { + data: GetProductEventsForTraceInput +}) { + const input = yield* decodeInput(ProductEventsForTraceInputSchema, data, "getProductEventsForTrace") + + const result = yield* runWarehouseQuery("productEventsForTrace", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventsForTrace({ + payload: new ProductEventsForTraceRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } +}) + +const ProductEventTraceSamplesInputSchema = Schema.Struct({ + ...TimeWindowFields, + eventName: Schema.String, + limit: Schema.optional(PositiveInt), +}) + +export type GetProductEventTraceSamplesInput = (typeof ProductEventTraceSamplesInputSchema)["Encoded"] + +export interface ProductEventTraceSample { + traceId: string + spanId: string + timestamp: string + serviceName: string + userId: string + visitorId: string +} + +export function getProductEventTraceSamples({ data }: { data: GetProductEventTraceSamplesInput }) { + return getProductEventTraceSamplesEffect({ data }) +} + +const getProductEventTraceSamplesEffect = Effect.fn("QueryEngine.getProductEventTraceSamples")( + function* ({ data }: { data: GetProductEventTraceSamplesInput }) { + const input = yield* decodeInput( + ProductEventTraceSamplesInputSchema, + data, + "getProductEventTraceSamples", + ) + + const result = yield* runWarehouseQuery("productEventTraceSamples", () => + Effect.gen(function* () { + const client = yield* MapleInternalAtomClient + return yield* client.queryEngine.productEventTraceSamples({ + payload: new ProductEventTraceSamplesRequest(input), + }) + }), + ) + + return { data: result.data satisfies ReadonlyArray } + }, +) + // Dashboard funnel widget (route data source `product_events_funnel`). // // The widget's stored `display.funnel` definition — steps, key, window, an diff --git a/apps/web/src/components/analytics/product-event-trace-samples.tsx b/apps/web/src/components/analytics/product-event-trace-samples.tsx new file mode 100644 index 000000000..5f94df68d --- /dev/null +++ b/apps/web/src/components/analytics/product-event-trace-samples.tsx @@ -0,0 +1,96 @@ +import { Link } from "@tanstack/react-router" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { productEventTraceSamplesResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { formatTimestampInTimezone } from "@/lib/timezone-format" +import { useTimezonePreference } from "@/hooks/use-timezone-preference" +import { ChartBarTrendUpIcon } from "@/components/icons" + +/** + * Recent traces behind one product event — the other half of the link a + * `maple.product_event.name` span attribute creates. + * + * Only events annotated on a span can answer this. A `track()` call from the + * browser and a `POST /v1/events` row carry no trace, so for those the list is + * empty and this renders nothing rather than an empty state: "no traces" is not + * a finding about the event, it just means the event was not emitted from a + * span, and saying so on every browser event would be noise on the majority of + * them. + * + * A FAILURE is not silent, though, unlike the trace-page panel. This one mounts + * because the user explicitly filtered to an event and asked for it, and empty + * is a meaningful answer here — so swallowing an error would answer their + * question wrongly ("this event has no traces") with no way to tell and no way + * to retry. + */ +export function ProductEventTraceSamples({ + eventName, + startTime, + endTime, +}: { + eventName: string + startTime: string + endTime: string +}) { + const { effectiveTimezone } = useTimezonePreference() + const result = useAtomValue( + productEventTraceSamplesResultAtom({ data: { eventName, startTime, endTime, limit: 10 } }), + ) + + return Result.builder(result) + .onSuccess((response) => { + if (response.data.length === 0) return null + return ( +
+
+ +

Traces behind “{eventName}”

+
+
    + {/* Index included: at-least-once ingest can duplicate a row, and + one trace can fire the event from several spans. */} + {response.data.map((sample, index) => ( +
  • + + + {sample.traceId.slice(0, 8)} + + {sample.serviceName === "" ? null : {sample.serviceName}} + {sample.userId || sample.visitorId ? ( + + · {sample.userId || sample.visitorId} + + ) : null} + + {formatTimestampInTimezone(sample.timestamp, { + timeZone: effectiveTimezone, + })} + + +
    • + ))} +
+
+ ) + }) + .onError(() => ( +
+
+ +

Traces behind “{eventName}”

+
+

+ Could not load traces for this event. This is a query failure, not an empty result — + reload to try again. +

+
+ )) + .orElse(() => null) +} diff --git a/apps/web/src/components/traces/trace-product-events.tsx b/apps/web/src/components/traces/trace-product-events.tsx new file mode 100644 index 000000000..e2f10b7cf --- /dev/null +++ b/apps/web/src/components/traces/trace-product-events.tsx @@ -0,0 +1,153 @@ +import { formatWarehouseDateTime, parseWarehouseDateTime } from "@maple/query-engine" +import { Result, useAtomValue } from "@/lib/effect-atom" +import { productEventsForTraceResultAtom } from "@/lib/services/atoms/warehouse-query-atoms" +import { ChartBarTrendUpIcon } from "@/components/icons" +import { Badge } from "@maple/ui/components/ui/badge" +import type { TraceProductEvent } from "@/api/warehouse/product-events" + +/** Same margin and reasoning as `TraceLogsLink`: clock skew between services can + * stamp an annotated span slightly outside the root span's own window, and the + * bound is also what keeps the lookup off every retained partition. */ +const WINDOW_MARGIN_MS = 5 * 60 * 1000 + +/** + * The product events this trace produced — spans the team annotated in their own + * code with `maple.product_event.name`. + * + * Renders nothing while loading, on failure, and when the trace produced none, + * which is the overwhelming majority of traces. That silence is the whole design + * of the panel: it sits unconditionally in the trace body and only appears on + * the traces where a request actually accomplished something the business + * counts, so it reads as a finding rather than as another empty section. + * + * Clicking an event selects its span in the waterfall, which is the point of the + * link — "the conversion happened, and here is the code path that did it." + */ +export function TraceProductEvents({ + traceId, + traceStartTime, + totalDurationMs, + onSelectSpan, +}: { + traceId: string + traceStartTime: string + totalDurationMs: number + onSelectSpan: (spanId: string) => void +}) { + const traceStartMs = parseWarehouseDateTime(traceStartTime) + if (Number.isNaN(traceStartMs)) return null + return ( + + ) +} + +/** + * Split out so the unparseable-timestamp case never constructs an atom key at + * all. Folding the guard in after the `useAtomValue` — passing `?? ""` to + * satisfy the required time fields — mounts the atom, fails `decodeInput` + * against `TinybirdDateTime`, and exports a `QueryEngine.getProductEventsForTrace` + * failure span on every render, for a query nobody wanted. It happened not to + * reach the network only because that schema rejects `""`; a later change making + * the window optional-with-fallback (which the logs client already does) would + * have turned it into a real full-window warehouse read. + */ +function LoadedTraceProductEvents({ + traceId, + startTime, + endTime, + onSelectSpan, +}: { + traceId: string + startTime: string + endTime: string + onSelectSpan: (spanId: string) => void +}) { + const result = useAtomValue(productEventsForTraceResultAtom({ data: { traceId, startTime, endTime } })) + + return Result.builder(result) + .onSuccess((response) => { + if (response.data.length === 0) return null + return ( +
+
+ +

Product events

+ {response.data.length} +
+
    + {/* Index, not spanId+eventName: `SpanId` is '' on any row that + reached the table without a span, so two same-named events in + one trace collide on that key — and at-least-once ingest can + duplicate a row outright. The list is ordered by the server + and never reordered client-side, so the index is stable. */} + {response.data.map((event, index) => ( + + ))} +
+
+ ) + }) + .onError(() => null) + .orElse(() => null) +} + +function ProductEventRow({ + event, + onSelectSpan, +}: { + event: TraceProductEvent + onSelectSpan: (spanId: string) => void +}) { + // UserId first, then GroupId, then VisitorId — the same precedence the funnel + // person key uses, so the identity shown here is the one the event will be + // counted under rather than whichever field happened to be set. + const person = event.userId || event.groupId || event.visitorId + const props = Object.entries(event.attributes) + + const content = ( + <> + {event.eventName} + {event.serviceName === "" ? null : ( + {event.serviceName} + )} + {person === "" ? null : · {person}} + {props.map(([key, value]) => ( + + {key}: {value} + + ))} + + ) + const rowClassName = "flex w-full flex-wrap items-center gap-x-2 gap-y-1 px-3 py-2 text-left text-xs" + + // A row with no span to select is a plain row, not a disabled button. As a + // disabled button its whole content — event name, identity, props — leaves + // the tab order and is unreachable by keyboard, while a mouse user gets no + // cue at all: there is no dimming, only a hover highlight that silently + // doesn't appear. The information is worth reading either way; only the + // navigation is unavailable. + if (event.spanId === "") { + return
  • {content}
    • + } + + return ( +
  • + +
    • + ) +} diff --git a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts index b3023b3cf..de9d9ee2d 100644 --- a/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts +++ b/apps/web/src/lib/services/atoms/warehouse-query-atoms.ts @@ -123,7 +123,11 @@ import { getWebAnalyticsSummary, getWebAnalyticsTimeseries, } from "@/api/warehouse/web-analytics" -import { getProductEventNames } from "@/api/warehouse/product-events" +import { + getProductEventNames, + getProductEventsForTrace, + getProductEventTraceSamples, +} from "@/api/warehouse/product-events" /** * The error union every warehouse server function fails with: the structured @@ -385,6 +389,16 @@ export const productEventNamesResultAtom = makeQueryAtomFamily(getProductEventNa staleTime: 60_000, }) +// A completed trace's product events never change, so this is only ever refetched +// because the trace is still open. 60s matches the route cache behind it. +export const productEventsForTraceResultAtom = makeQueryAtomFamily(getProductEventsForTrace, { + staleTime: 60_000, +}) + +export const productEventTraceSamplesResultAtom = makeQueryAtomFamily(getProductEventTraceSamples, { + staleTime: 60_000, +}) + export const getReplayResultAtom = makeQueryAtomFamily(getReplay, { staleTime: 60_000, }) diff --git a/apps/web/src/routes/analytics/index.tsx b/apps/web/src/routes/analytics/index.tsx index a95d86894..f24e2d6b6 100644 --- a/apps/web/src/routes/analytics/index.tsx +++ b/apps/web/src/routes/analytics/index.tsx @@ -19,6 +19,7 @@ import { type BreakdownDimension, } from "@/components/analytics/analytics-breakdown-panel" import { AnalyticsBotNotice } from "@/components/analytics/analytics-bot-notice" +import { ProductEventTraceSamples } from "@/components/analytics/product-event-trace-samples" import { AnalyticsFilterSidebar } from "@/components/analytics/analytics-filter-sidebar" import { AnalyticsLiveBadge } from "@/components/analytics/analytics-live-badge" import { @@ -588,6 +589,13 @@ function AnalyticsContent({ { id: "events", dimensions: eventDimensions, wide: true }, ] + // The reverse of the trace view's product-events panel, shown at the + // one moment it is asked for: the user has filtered to a single event + // and now wants to see requests that fired it. Renders nothing unless + // the event came from an annotated span, so browser `track()` events + // simply do not grow a section. + const eventName = filters.eventName + return (
    {cards.map((card) => ( @@ -603,6 +611,15 @@ function AnalyticsContent({ />
    ))} + {eventName === undefined ? null : ( +
    + +
    + )} ) }) diff --git a/apps/web/src/routes/traces/$traceId.tsx b/apps/web/src/routes/traces/$traceId.tsx index 2b2a51319..8cbb4fb3a 100644 --- a/apps/web/src/routes/traces/$traceId.tsx +++ b/apps/web/src/routes/traces/$traceId.tsx @@ -13,6 +13,7 @@ import { QueryErrorState } from "@/components/common/query-error-state" import { TraceViewTabs } from "@maple/ui/components/traces/trace-view-tabs" import { SpanDetailPanel } from "@/components/traces/span-detail-panel" import { TraceAnatomyStrip } from "@/components/traces/trace-anatomy-strip" +import { TraceProductEvents } from "@/components/traces/trace-product-events" import { Skeleton } from "@maple/ui/components/ui/skeleton" import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "@maple/ui/components/ui/resizable" import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@maple/ui/components/ui/sheet" @@ -245,6 +246,20 @@ function TraceDetailContent({ [search.spanId, navigate], ) + // The product-events panel knows a span ID, not a `SpanNode` — its rows come + // from `product_events`, not the hierarchy. Selecting by id keeps the two + // sources from having to agree on a node shape. + const handleSelectSpanId = React.useCallback( + (spanId: string) => { + if (search.spanId === spanId) return + navigate({ + search: (prev: Record) => ({ ...prev, spanId }), + replace: true, + }) + }, + [search.spanId, navigate], + ) + const handleCloseSpanDetails = React.useCallback(() => { navigate({ search: (prev: Record) => ({ ...prev, spanId: undefined }), @@ -335,6 +350,13 @@ function TraceDetailContent({ commitSha={commitSha} /> + + {isMobile ? ( // A 60/40 side-by-side split leaves each pane ~150px on a phone. Give the waterfall // the full width and float the span detail over it instead. diff --git a/docs/product-events-funnels.md b/docs/product-events-funnels.md index fc1e135f0..cc4addf63 100644 --- a/docs/product-events-funnels.md +++ b/docs/product-events-funnels.md @@ -277,3 +277,150 @@ the `product_events_funnel` route params (`ProductEventsFunnelWidgetParams` in ` which the browser server function and the share API's route plan both decode. With a breakdown the route answers `{ name, value, group }` rows (top 6 groups by step-1 count) and the funnel chart draws one bar per group per step with a legend. + +## Product events from traces — annotate in code (2026-09) + +The fourth feed into `product_events`, after browser (`session_events` MV), server and mobile +(`POST /v1/events`). A team marks a span they already emit and it becomes a funnel step that +links back to the request that performed it. + +```ts +span.setAttributes({ + "maple.product_event.name": "checkout_completed", // required — presence is the predicate + "maple.product_event.user_id": user.id, // optional identity + "maple.product_event.group_id": org.id, + "maple.product_event.visitor_id": anonId, + "maple.product_event.url": req.url, // optional page context +}) +``` + +**Every other attribute on the span becomes an event property by default.** Nothing has to be +declared to get started — whatever the team already sets on the span (`plan`, `order.total`, the +full HTTP/DB semconv surface) lands in `Attributes` and is available to funnel breakdowns. The +`maple.product_event.*` control keys are stripped, since they are already promoted to their own +columns. + +Two optional controls narrow or replace that default. Both are themselves span attributes: a +materialized view is static SQL per cluster and has no per-org config to read. + +```ts +"maple.product_event.include": "plan,seats" // ONLY these span keys (whitespace trimmed) +"maple.product_event.prop.plan": "pro" // explicit prop, merged over the base, wins ties +"maple.product_event.include": "" // and together: full overwrite +``` + +Three tiers out of one mechanism rather than three modes to pick between: + +| `include` | `prop.*` | `Attributes` | +| --- | --- | --- | +| absent | — | every span attribute | +| absent | set | every span attribute, with the props overriding on a key collision | +| `"plan,seats"` | — | only `plan` and `seats` | +| `""` | set | only the props — the overwrite case | + +`include` switches on **key presence**, not on a non-empty value, which is what makes the empty +string mean "no span attributes" rather than "no filter". There is no separate replace flag to get +wrong, and `mapUpdate(base, props)` argument order is the override rule — swapped, an override +would be discarded exactly when the key it meant to correct was already present. + +Verified against ClickHouse 26.2, all three tiers: + +| Scenario | Span attributes | Result | +| --- | --- | --- | +| default | `http.method`, `plan=free`, `seats=5`, `prop.plan=pro` | `{http.method, seats, plan:'pro'}` | +| `include: "plan, seats"` | + `noise` | `{plan:'free', seats:'5'}` | +| `include: ""` + `prop.plan=pro` | `http.method`, `plan=free` | `{plan:'pro'}` | + +The contract lives in one place — `packages/domain/src/tinybird/product-event-attributes.ts` — +and is read by exactly two consumers that must agree byte for byte: `productEventsTracesMv` +(managed orgs, via `tinybird deploy`) and the frozen copy inside ClickHouse migration 0026 (BYO +clusters). The migration's copy is deliberately NOT imported from the constant: a delta migration +describes one step in history, and a shared constant would silently rewrite what 0026 did the next +time the live projection changes. + +### Why an attribute and not a UI action + +A product event has to be emitted by the code path that performed the thing, at the moment it +performed it. Marking a trace by hand in the UI marks *one sampled trace*, cannot be replayed over +history, and puts a mutable user-authored row into an append-only fact table. An attribute marks +every trace the path produces, applies retroactively across the whole `traces` retention window, +and is reviewable in the customer's own diff. There is no second store and no write path from the +dashboard — the span is the record, the product event is its projection. + +### The link + +`product_events` gained `TraceId`/`SpanId` (migration 0026, `DEFAULT ''`, appended, plus a +`bloom_filter` on `TraceId`). Non-empty only on `Source = 'trace'` rows. Real columns rather than +`Attributes` keys because both directions filter on them, and a `Map` lookup on this table reads +the whole map per row — the exact cost `product_events` was split out of `session_events` to avoid. + +| Direction | Query | Surface | +| --- | --- | --- | +| trace → its product events | `productEventsForTraceQuery` | trace detail page, under the anatomy strip | +| event → the traces behind it | `productEventTraceSamplesQuery` | `/analytics`, when the `eventName` filter is set | + +Both are `profile: "list"` with a flat `cache: 60` rather than `timeRangeCache`: they are point +lookups whose answer does not widen with the range asked about, and a completed trace's events +never change at all. + +### What it costs + +The MV predicate is one `Map` value read per incoming span, on the same block every other `traces` +MV already fires on. An MV sees the insert block, not the table, so `idx_span_attr_keys` does not +help it — this is a real, deliberately small per-span ingest cost. + +Copying the whole `SpanAttributes` map **by default** is the deliberate expensive choice. A server +span's map is dominated by HTTP/DB semconv keys, and `product_events` keeps 365 days against raw +`traces`' 30 — so an annotated span's attributes outlive the span itself by a factor of twelve. What +the default buys is that nothing has to be declared to get a useful event; `include` is the lever +for a team that has measured the cost and wants it back, and it is a one-line change on the span +rather than a schema migration. + +The practical consequence to watch: attribute pickers over product events list the span's whole +semconv surface for any team that has not set `include`. If that becomes the dominant cost across +orgs rather than for one of them, the lever is a per-org key denylist at the MV — the per-span +`include` handles the single-team case already. + +The whole `Attributes` expression only evaluates for rows passing the `WHERE`, i.e. annotated spans, +so its cost is paid per product event rather than per span. The predicate itself stays one map +lookup. + +### Rollout + +1. **Managed**: `bun run --cwd apps/api tinybird:deploy` creates `product_events_traces_mv` and + adds the two columns, then an explicit `tb` populate from `traces` (bounded by its 30-day TTL). + Blocked on the same manual step the rest of this document's checklist is — see + `project_product_events_tinybird_rollout_pending`. + + **The populate is one-shot and overlap-prone.** Unlike BYO and local, the managed surface has no + `DELETE WHERE Source = 'trace'` step, so running it twice double-inserts, and running it after + the MV is already live double-counts every annotated span ingested between MV creation and the + populate's own snapshot. BYO risks a gap; managed risks duplicates. Same caveat 0014 and 0021 + accepted — but on a table feeding customer-visible funnels, a double-counted conversion is worse + than a missing one. Populate once, immediately after deploy, and if it fails partway prefer + deleting the trace rows by hand over re-running it blind. +2. **BYO ClickHouse**: migration 0026, `requiredForIngest: false`. That is safe for one reason + worth knowing before anyone touches `datasources.ts`: `TraceId`/`SpanId` are declared with **no + `jsonPath`**, so the insert-mapping generator omits them and the Rust gateway's + `INSERT INTO product_events (…)` never names them — a cluster stamped below 26 still accepts + every row it sends. Give those columns a `jsonPath` and the flag becomes a data-loss bug: the + readiness gate still says 21, so unmigrated BYO orgs keep routing to their own cluster, where + the INSERT fails on the unknown column, retries, trips the breaker and drops the batch. + Backfills the trace half from `traces` itself. +3. **Local CLI**: local schema v13 → v14, same backfill. + +Both BYO and local scope their idempotency `DELETE` to `Timestamp >= (SELECT min(Timestamp) FROM +traces)` rather than deleting all trace rows. `product_events` keeps 365 days and `traces` 30, so an +unbounded delete on a *late* re-apply would clear a year of funnel history and rebuild only a month +of it. + +### Not in this cut + +- **No MCP tool.** `list_product_events` still returns names only, and `inspect_trace` does not + surface a trace's product events. An agent cannot walk the link yet; the queries and the HTTP + routes it would sit on both exist. +- **No SDK helper.** Teams set the attributes by hand on whatever span API they already use. A + `markProductEvent(span, name, { props, include })` in `@maple-dev/effect-sdk` would be the obvious + next step — it is a wrapper over `setAttributes` that builds the `prop.*` keys and joins + `include`, not new machinery, and it is where the empty-string overwrite idiom would get a name + (`attributes: "none"`) instead of being a documented convention. diff --git a/packages/domain/src/clickhouse/migrations/0026_product_events_from_traces.ts b/packages/domain/src/clickhouse/migrations/0026_product_events_from_traces.ts new file mode 100644 index 000000000..e3ef20f8f --- /dev/null +++ b/packages/domain/src/clickhouse/migrations/0026_product_events_from_traces.ts @@ -0,0 +1,222 @@ +import type { BackfillSpec } from "../backfill" + +/** + * Frozen copy of the trace→product-event projection as of this migration. + * + * NOT imported from `tinybird/product-event-attributes.ts`, deliberately, and + * for the reason migration 0019 spells out: a delta migration describes one step + * in history. A shared constant would silently rewrite what this migration did + * the next time the live projection changes, and a BYO cluster replaying the + * chain would land somewhere the chain never says it goes. + * + * Shared *within* this file between the materialized view and the backfill, + * which is the 0021 pattern — two copies of one SELECT is two chances for a + * backfilled span and a live span to disagree. + */ +const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + 'trace' AS Source, + SpanAttributes['session.id'] AS SessionId, + 0 AS Seq, + SpanAttributes['maple.product_event.visitor_id'] AS VisitorId, + SpanAttributes['maple.product_event.user_id'] AS UserId, + SpanAttributes['maple.product_event.group_id'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['maple.product_event.name'] AS EventName, + domain(SpanAttributes['maple.product_event.url']) AS Host, + path(SpanAttributes['maple.product_event.url']) AS PagePath, + SpanAttributes['maple.product_event.url'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, 'maple.product_event.') + AND ( + NOT has(mapKeys(SpanAttributes), 'maple.product_event.include') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['maple.product_event.include']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, 26), v), + mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +const PRODUCT_EVENTS_TRACE_FILTER = "SpanAttributes['maple.product_event.name'] != ''" + +/** + * Backfill of the trace half of {@link migration_0026_product_events_from_traces}. + * + * Row-wise, so any chunk boundary is safe. Idempotent by `DELETE WHERE Source = + * 'trace'` before it runs — same shape as 0021's browser half, and safe for the + * same reason: it clears only rows this projection owns, never a browser row + * from the other view or a directly ingested one that has no source to come back + * from. + * + * Bounded by `traces`' own 30-day TTL, which is all there is to rebuild from. + * `product_events` keeps 365 days, so an org that annotates spans today sees + * ~30 days of history immediately and accrues the rest going forward. + */ +export const productEventsTracesBackfill: BackfillSpec = { + kind: "backfill", + target: "product_events", + columns: [ + "OrgId", + "Timestamp", + "Source", + "SessionId", + "Seq", + "VisitorId", + "UserId", + "GroupId", + "Kind", + "EventName", + "Host", + "PagePath", + "Url", + "ServiceName", + "Attributes", + "TraceId", + "SpanId", + ], + from: "traces", + tsColumn: "Timestamp", + select: PRODUCT_EVENTS_TRACE_PROJECTION_SQL, + where: PRODUCT_EVENTS_TRACE_FILTER, +} + +/** + * Migration 0026 — product events annotated in code. + * + * A customer marks a span they already emit: + * + * ``` + * span.setAttributes({ "maple.product_event.name": "checkout_completed" }) + * ``` + * + * and it becomes a row in `product_events` that a funnel steps on, carrying the + * `TraceId` it came from. The annotation is instrumentation, not a UI action and + * not a second store: the span remains the record and the product event is its + * projection, so it applies to every trace the annotated code path produces + * rather than to the one trace someone happened to be looking at. + * + * Two things, in order: + * + * 1. `product_events` gains `TraceId`/`SpanId` (`DEFAULT ''`) plus a bloom + * filter on `TraceId`. Metadata-only `ADD COLUMN`s, appended, which is why + * they sit last in the schema and last in every projection. + * 2. `product_events_traces_mv` projects annotated spans in, and the trace half + * is backfilled from `traces`' 30-day window. + * + * `product_events_mv` is dropped and re-created rather than left alone: an MV's + * SELECT is fixed at creation, so the pre-0026 body writes 15 columns into a + * 17-column table and every browser row inserted between the ALTER and the + * re-create would take defaults for the two new columns. They default to `''`, + * which is the correct value for a browser row — so this is ordering hygiene + * rather than a repair, and the re-create is what keeps the view's text and the + * table's schema describing the same thing. + * + * Re-runnable by construction: `ADD COLUMN IF NOT EXISTS`, `ADD INDEX IF NOT + * EXISTS`, and a `DELETE` scoped to the backfill's own source window so it + * clears exactly and only what the following backfill re-inserts — at any point + * in the table's life, not just on first apply. + * + * **BYO ClickHouse only.** Managed orgs get the same view via `tinybird deploy` + * from `materializations.ts`, with the populate as an explicit `tb` step at + * deploy time (see 0014 and 0021 — the SDK has no populate option). + * + * `requiredForIngest: false`, and that rests on ONE fact worth stating plainly + * because it is not local to this file: `TraceId`/`SpanId` are declared in + * `datasources.ts` WITHOUT a `jsonPath`, so + * `scripts/generate-clickhouse-insert-mappings.ts` omits them and the Rust + * gateway's `INSERT INTO product_events (…)` never names them. A cluster stamped + * below 24 therefore still accepts every row the gateway sends. + * + * Give those columns a `jsonPath` and this flag becomes a data-loss bug: the + * readiness gate compares `stamped >= clickHouseSchemaVersion`, which stays at + * 21, so a BYO org that has not applied 0026 is still routed to its own cluster + * — where the INSERT fails on the unknown column, retries, trips the breaker and + * drops the batch. The gate's safety argument is "an older binary writing into a + * newer schema", and that would be the inverse. + * + * Bumping `clickHouseSchemaVersion` instead would un-ready ingest routing for + * every BYO-CH org over a feature none of their existing writers touch, which is + * why the column declaration is the right place to solve it. + */ +export const migration_0026_product_events_from_traces = { + version: 26, + description: + "Add TraceId/SpanId to product_events and materialize product events from spans carrying the maple.product_event.name attribute", + requiredForIngest: false, + statements: [ + "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS TraceId String DEFAULT ''", + "ALTER TABLE product_events ADD COLUMN IF NOT EXISTS SpanId String DEFAULT ''", + "ALTER TABLE product_events ADD INDEX IF NOT EXISTS idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", + // The browser view is dropped and IMMEDIATELY recreated, before anything + // slow runs. 0021 bracketed its backfill with the drop because there the + // backfill WAS the browser feed and two writers of the same rows had to be + // impossible. Here the backfill reads `traces` and the browser view reads + // `session_events`, so the bracket buys nothing and costs everything: the + // backfill is chunked across up to 400 durable workflow steps over 30 days + // of spans, and every `session_events` navigation row ingested while the + // view is gone is never projected at all. That is a permanent hole in page + // views, not the "ordering hygiene" an earlier draft of this comment + // called it. Recreated here, the outage is one statement wide. + "DROP VIEW IF EXISTS product_events_mv", + // Frozen copy of the browser projection as of 0021, plus the two new + // columns explicitly at `''`. Same freezing rule as above: this is what + // the view's body was at this point in history. + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS +SELECT + OrgId, + Timestamp, + 'browser' AS Source, + SessionId, + Seq, + VisitorId, + UserId, + GroupId, + Type AS Kind, + if(Type = 'navigation', '$pageview', Message) AS EventName, + domain(Url) AS Host, + path(Url) AS PagePath, + Url, + '' AS ServiceName, + Attributes, + '' AS TraceId, + '' AS SpanId +FROM session_events +WHERE Type IN ('navigation', 'custom')`, + "DROP VIEW IF EXISTS product_events_traces_mv", + // Idempotency for the trace half only — never a browser row, never a + // directly ingested one. + // + // BOUNDED BY THE BACKFILL'S OWN WINDOW, which is what makes the claim + // above true on a LATE re-run rather than only on first apply. `traces` + // keeps 30 days and `product_events` keeps 365, so by day 100 the table + // holds trace rows the backfill can no longer rebuild. An unbounded + // `DELETE WHERE Source = 'trace'` would clear all 100 days and re-insert + // 30 — silently destroying 70 days of a customer's funnel history on a + // re-apply after a lost bookkeeping row. Scoping the delete to + // `min(Timestamp)` of the source means it removes exactly the rows the + // following backfill is about to write, and nothing older. + "DELETE FROM product_events WHERE Source = 'trace' AND Timestamp >= (SELECT min(Timestamp) FROM traces)", + productEventsTracesBackfill, + `CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS +SELECT +${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} +FROM traces +WHERE ${PRODUCT_EVENTS_TRACE_FILTER}`, + ], +} as const diff --git a/packages/domain/src/clickhouse/migrations/index.test.ts b/packages/domain/src/clickhouse/migrations/index.test.ts index 415512ddc..a251821e3 100644 --- a/packages/domain/src/clickhouse/migrations/index.test.ts +++ b/packages/domain/src/clickhouse/migrations/index.test.ts @@ -31,6 +31,7 @@ import { migration_0022_service_map_edge_quantiles } from "./0022_service_map_ed import { migration_0023_service_operations_discriminators } from "./0023_service_operations_discriminators" import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_revision" +import { migration_0026_product_events_from_traces } from "./0026_product_events_from_traces" import { migration_0021_product_events } from "./0021_product_events" import { clickHouseSchemaVersion, latestMigrationVersion, migrations } from "./index" @@ -47,10 +48,10 @@ const renderedSql = migration_0004_service_namespace_projections.statements describe("ClickHouse migrations", () => { it("keeps migrations ordered by version", () => { expect(migrations.map((m) => m.version)).toEqual([ - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, ]) - expect(migrations.at(-1)).toBe(migration_0025_commit_sha_vcs_revision) - expect(latestMigrationVersion).toBe(25) + expect(migrations.at(-1)).toBe(migration_0026_product_events_from_traces) + expect(latestMigrationVersion).toBe(26) // 0010 and 0014-0020 are read-path only and skipped by the ingest-gating // version; 0021 is not — the gateway writes `session_events`' new identity // columns and `product_events` directly, so a BYO-CH org must apply it @@ -73,6 +74,7 @@ describe("ClickHouse migrations", () => { expect(migration_0023_service_operations_discriminators.requiredForIngest).toBe(false) expect(migration_0024_ai_trace_index.requiredForIngest).toBe(false) expect(migration_0025_commit_sha_vcs_revision.requiredForIngest).toBe(false) + expect(migration_0026_product_events_from_traces.requiredForIngest).toBe(false) }) it("recreates both error-events MVs with the 4xx guard and the widened frame redaction", () => { diff --git a/packages/domain/src/clickhouse/migrations/index.ts b/packages/domain/src/clickhouse/migrations/index.ts index 5ef49414f..3673ef1a0 100644 --- a/packages/domain/src/clickhouse/migrations/index.ts +++ b/packages/domain/src/clickhouse/migrations/index.ts @@ -24,6 +24,7 @@ import { migration_0022_service_map_edge_quantiles } from "./0022_service_map_ed import { migration_0023_service_operations_discriminators } from "./0023_service_operations_discriminators" import { migration_0024_ai_trace_index } from "./0024_ai_trace_index" import { migration_0025_commit_sha_vcs_revision } from "./0025_commit_sha_vcs_revision" +import { migration_0026_product_events_from_traces } from "./0026_product_events_from_traces" /** * A migration statement is either a raw SQL string (structural DDL) or a @@ -80,6 +81,7 @@ export const migrations: ReadonlyArray = [ migration_0023_service_operations_discriminators, migration_0024_ai_trace_index, migration_0025_commit_sha_vcs_revision, + migration_0026_product_events_from_traces, ] as const /** Highest migration `version` bundled — i.e. the schema level a fully-applied diff --git a/packages/domain/src/generated/clickhouse-schema.ts b/packages/domain/src/generated/clickhouse-schema.ts index 0fa6fe904..f066ca8da 100644 --- a/packages/domain/src/generated/clickhouse-schema.ts +++ b/packages/domain/src/generated/clickhouse-schema.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-clickhouse-schema.ts // Do not edit manually. -export const projectRevision = "bf3419c18e581ffab5fa4e24aa56f421ac9c3d5a1ce734596747f2331d7d2ae0" as const +export const projectRevision = "4545ddafc64d7bd0234a65c2442cb615373026c6fcf1d2f4b3f0f0e6b5ca88d9" as const export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS ai_trace_index (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n TraceId String,\n SessionId String,\n VendorId LowCardinality(String),\n ServiceName LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, TraceId)\nTTL toDate(Timestamp) + INTERVAL 30 DAY", @@ -19,7 +19,7 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE TABLE IF NOT EXISTS metrics_gauge (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String))\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_histogram (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Count UInt64,\n Sum Float64,\n BucketCounts Array(UInt64),\n ExplicitBounds Array(Float64),\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n Flags UInt32,\n Min Nullable(Float64),\n Max Nullable(Float64),\n AggregationTemporality Int32\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", "CREATE TABLE IF NOT EXISTS metrics_sum (\n OrgId LowCardinality(String),\n ResourceAttributes Map(LowCardinality(String), String),\n ResourceSchemaUrl String,\n ScopeName String,\n ScopeVersion String,\n ScopeAttributes Map(LowCardinality(String), String),\n ScopeSchemaUrl String,\n ServiceName LowCardinality(String),\n MetricName LowCardinality(String),\n MetricDescription LowCardinality(String),\n MetricUnit LowCardinality(String),\n Attributes Map(LowCardinality(String), String),\n StartTimeUnix DateTime64(9),\n TimeUnix DateTime64(9),\n Value Float64,\n Flags UInt32,\n ExemplarsTraceId Array(String),\n ExemplarsSpanId Array(String),\n ExemplarsTimestamp Array(DateTime64(9)),\n ExemplarsValue Array(Float64),\n ExemplarsFilteredAttributes Array(Map(LowCardinality(String), String)),\n AggregationTemporality Int32,\n IsMonotonic Bool\n)\nENGINE = MergeTree\nPARTITION BY toDate(TimeUnix)\nORDER BY (OrgId, ServiceName, MetricName, Attributes, toUnixTimestamp64Nano(TimeUnix))\nTTL toDate(TimeUnix) + INTERVAL 90 DAY", - "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", + "CREATE TABLE IF NOT EXISTS product_events (\n OrgId LowCardinality(String),\n Timestamp DateTime64(9),\n Source LowCardinality(String) DEFAULT 'browser',\n SessionId String DEFAULT '',\n Seq UInt32 DEFAULT 0,\n VisitorId String DEFAULT '',\n UserId String DEFAULT '',\n GroupId String DEFAULT '',\n Kind LowCardinality(String),\n EventName String,\n Host LowCardinality(String) DEFAULT '',\n PagePath String DEFAULT '',\n Url String DEFAULT '',\n ServiceName LowCardinality(String) DEFAULT '',\n Attributes Map(String, String) DEFAULT map(),\n TraceId String DEFAULT '',\n SpanId String DEFAULT '',\n INDEX idx_event_name EventName TYPE set(64) GRANULARITY 4,\n INDEX idx_user_id UserId TYPE bloom_filter GRANULARITY 4,\n INDEX idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, Timestamp, VisitorId, SessionId, Seq)\nTTL toDate(Timestamp) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_address_resolutions_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n SourceService LowCardinality(String),\n ParentServerAddress String,\n ResolvedTargetService LowCardinality(String),\n DeploymentEnv LowCardinality(String)\n)\nENGINE = ReplacingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, SourceService, ParentServerAddress, ResolvedTargetService)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_external_edges_hourly (\n OrgId LowCardinality(String),\n Hour DateTime,\n ServiceName LowCardinality(String),\n TargetType LowCardinality(String),\n TargetSystem LowCardinality(String),\n TargetName String,\n DeploymentEnv LowCardinality(String),\n CallCount SimpleAggregateFunction(sum, UInt64),\n ErrorCount SimpleAggregateFunction(sum, UInt64),\n DurationSumMs SimpleAggregateFunction(sum, Float64),\n MaxDurationMs SimpleAggregateFunction(max, Float64),\n SampleRateSum SimpleAggregateFunction(sum, Float64),\n DurationQuantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95), UInt64, UInt32)\n)\nENGINE = AggregatingMergeTree\nPARTITION BY toDate(Hour)\nORDER BY (OrgId, Hour, DeploymentEnv, ServiceName, TargetType, TargetSystem, TargetName)\nTTL toDate(Hour) + INTERVAL 365 DAY", "CREATE TABLE IF NOT EXISTS service_map_children (\n OrgId LowCardinality(String),\n Timestamp DateTime,\n TraceId String,\n ParentSpanId String,\n ServiceName LowCardinality(String),\n SpanKind LowCardinality(String),\n Duration UInt64,\n StatusCode LowCardinality(String),\n TraceState String,\n DeploymentEnv LowCardinality(String)\n)\nENGINE = MergeTree\nPARTITION BY toDate(Timestamp)\nORDER BY (OrgId, TraceId, ParentSpanId, Timestamp)\nTTL Timestamp + INTERVAL 30 DAY", @@ -57,7 +57,8 @@ export const latestSnapshotStatements: ReadonlyArray = [ "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_gauge_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'gauge' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_gauge\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_histogram_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'histogram' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n toUInt8(0) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_histogram\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", "CREATE MATERIALIZED VIEW IF NOT EXISTS metric_catalog_sum_mv TO metric_catalog AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(TimeUnix)) AS Hour,\n 'sum' AS MetricType,\n ServiceName,\n MetricName,\n anyLast(MetricDescription) AS MetricDescription,\n anyLast(MetricUnit) AS MetricUnit,\n anyLast(toUInt8(IsMonotonic)) AS IsMonotonic,\n count() AS DataPointCount,\n min(toDateTime(TimeUnix)) AS FirstSeen,\n max(toDateTime(TimeUnix)) AS LastSeen\n FROM metrics_sum\n GROUP BY OrgId, Hour, MetricType, ServiceName, MetricName", - "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes,\n '' AS TraceId,\n '' AS SpanId\n FROM session_events\n WHERE Type IN ('navigation', 'custom')", + "CREATE MATERIALIZED VIEW IF NOT EXISTS product_events_traces_mv TO product_events AS\nSELECT\n OrgId,\n Timestamp,\n 'trace' AS Source,\n SpanAttributes['session.id'] AS SessionId,\n 0 AS Seq,\n SpanAttributes['maple.product_event.visitor_id'] AS VisitorId,\n SpanAttributes['maple.product_event.user_id'] AS UserId,\n SpanAttributes['maple.product_event.group_id'] AS GroupId,\n 'custom' AS Kind,\n SpanAttributes['maple.product_event.name'] AS EventName,\n domain(SpanAttributes['maple.product_event.url']) AS Host,\n path(SpanAttributes['maple.product_event.url']) AS PagePath,\n SpanAttributes['maple.product_event.url'] AS Url,\n ServiceName,\n mapUpdate(\n CAST(\n mapFilter(\n (k, v) -> NOT startsWith(k, 'maple.product_event.')\n AND (\n NOT has(mapKeys(SpanAttributes), 'maple.product_event.include')\n OR has(\n arrayMap(\n key -> trimBoth(key),\n splitByChar(',', SpanAttributes['maple.product_event.include'])\n ),\n k\n )\n ),\n SpanAttributes\n ),\n 'Map(String, String)'\n ),\n mapApply(\n (k, v) -> (substring(k, 26), v),\n mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes)\n )\n ) AS Attributes,\n TraceId,\n SpanId\n FROM traces\n WHERE SpanAttributes['maple.product_event.name'] != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_external_edges_hourly_mv TO service_external_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', 'messaging',\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', 'rpc',\n 'http'\n ) AS TargetType,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '', SpanAttributes['messaging.system'],\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '', SpanAttributes['rpc.system'],\n ''\n ) AS TargetSystem,\n multiIf(\n coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '' OR SpanAttributes['messaging.system'] != '',\n if(coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != '', coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']), SpanAttributes['messaging.system']),\n SpanAttributes['rpc.service'] != '' OR SpanAttributes['rpc.system'] != '',\n if(SpanAttributes['rpc.service'] != '', SpanAttributes['rpc.service'], SpanAttributes['rpc.system']),\n if(SpanAttributes['server.address'] != '',\n SpanAttributes['server.address'],\n if(SpanAttributes['http.host'] != '',\n SpanAttributes['http.host'],\n SpanAttributes['url.authority']))\n ) AS TargetName,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n sum(SampleRate) AS SampleRateSum,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND SpanAttributes['db.system.name'] = ''\n AND ServiceName != ''\n AND (\n SpanAttributes['server.address'] != ''\n OR SpanAttributes['http.host'] != ''\n OR SpanAttributes['url.authority'] != ''\n OR coalesce(nullIf(SpanAttributes['messaging.destination.name'], ''), SpanAttributes['messaging.destination']) != ''\n OR SpanAttributes['messaging.system'] != ''\n OR SpanAttributes['rpc.service'] != ''\n OR SpanAttributes['rpc.system'] != ''\n )\n GROUP BY OrgId, Hour, ServiceName, TargetType, TargetSystem, TargetName, DeploymentEnv\n HAVING TargetName != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_children_mv TO service_map_children AS\nSELECT\n OrgId,\n toDateTime(Timestamp) AS Timestamp,\n TraceId,\n ParentSpanId,\n ServiceName,\n SpanKind,\n Duration,\n StatusCode,\n TraceState,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv\n FROM traces\n WHERE SpanKind IN ('Server', 'Consumer')\n AND ParentSpanId != ''", "CREATE MATERIALIZED VIEW IF NOT EXISTS service_map_db_edges_hourly_mv TO service_map_db_edges_hourly AS\nSELECT\n OrgId,\n toStartOfHour(toDateTime(Timestamp)) AS Hour,\n ServiceName,\n coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) AS DbSystem,\n if(match(coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name']), '^([0-9a-fA-F]{32}|.*[.]hyperdrive[.]local)$'), 'hyperdrive', coalesce(nullIf(SpanAttributes['db.namespace'], ''), nullIf(SpanAttributes['db.name'], ''), nullIf(SpanAttributes['server.address'], ''), SpanAttributes['net.peer.name'])) AS DbNamespace,\n coalesce(nullIf(ResourceAttributes['deployment.environment.name'], ''), ResourceAttributes['deployment.environment']) AS DeploymentEnv,\n count() AS CallCount,\n countIf(StatusCode = 'Error') AS ErrorCount,\n sum(Duration / 1000000) AS DurationSumMs,\n max(Duration / 1000000) AS MaxDurationMs,\n countIf(TraceState LIKE '%th:%') AS SampledSpanCount,\n countIf(TraceState = '' OR TraceState NOT LIKE '%th:%') AS UnsampledSpanCount,\n sum(SampleRate) AS SampleRateSum,\n quantilesTDigestWeightedState(0.5, 0.95)(Duration, toUInt32(greatest(SampleRate, 1.0))) AS DurationQuantiles\n FROM traces\n WHERE SpanKind IN ('Client', 'Producer')\n AND coalesce(nullIf(SpanAttributes['db.system.name'], ''), SpanAttributes['db.system']) != ''\n AND ServiceName != ''\n GROUP BY OrgId, Hour, ServiceName, DbSystem, DbNamespace, DeploymentEnv", diff --git a/packages/domain/src/generated/tinybird-project-manifest.ts b/packages/domain/src/generated/tinybird-project-manifest.ts index f2da6a254..99568dba7 100644 --- a/packages/domain/src/generated/tinybird-project-manifest.ts +++ b/packages/domain/src/generated/tinybird-project-manifest.ts @@ -1,7 +1,7 @@ // This file is generated by scripts/generate-tinybird-project-manifest.ts // Do not edit manually. -export const projectRevision = "bf3419c18e581ffab5fa4e24aa56f421ac9c3d5a1ce734596747f2331d7d2ae0" as const +export const projectRevision = "4545ddafc64d7bd0234a65c2442cb615373026c6fcf1d2f4b3f0f0e6b5ca88d9" as const export const datasources = [ { @@ -82,7 +82,7 @@ export const datasources = [ { name: "product_events", content: - "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map()\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4", + "DESCRIPTION >\n Product events fact table: browser page views and track() calls (materialized from session_events) plus events posted directly by backends and mobile apps via POST /v1/events. Carries the person key (VisitorId/UserId/GroupId). Powers page views, top pages and funnels.\n\nSCHEMA >\n OrgId LowCardinality(String) `json:$.org_id`,\n Timestamp DateTime64(9) `json:$.timestamp`,\n Source LowCardinality(String) `json:$.source` DEFAULT 'browser',\n SessionId String `json:$.session_id` DEFAULT '',\n Seq UInt32 `json:$.seq` DEFAULT 0,\n VisitorId String `json:$.visitor_id` DEFAULT '',\n UserId String `json:$.user_id` DEFAULT '',\n GroupId String `json:$.group_id` DEFAULT '',\n Kind LowCardinality(String) `json:$.kind`,\n EventName String `json:$.event_name`,\n Host LowCardinality(String) `json:$.host` DEFAULT '',\n PagePath String `json:$.page_path` DEFAULT '',\n Url String `json:$.url` DEFAULT '',\n ServiceName LowCardinality(String) `json:$.service_name` DEFAULT '',\n Attributes Map(String, String) `json:$.attributes` DEFAULT map(),\n TraceId String `json:$.TraceId` DEFAULT '',\n SpanId String `json:$.SpanId` DEFAULT ''\n\nENGINE \"MergeTree\"\nENGINE_PARTITION_KEY \"toDate(Timestamp)\"\nENGINE_SORTING_KEY \"OrgId, Timestamp, VisitorId, SessionId, Seq\"\nENGINE_TTL \"toDate(Timestamp) + INTERVAL 365 DAY\"\n\nINDEXES >\n idx_event_name EventName TYPE set(64) GRANULARITY 4\n idx_user_id UserId TYPE bloom_filter GRANULARITY 4\n idx_trace_id TraceId TYPE bloom_filter GRANULARITY 4", }, { name: "service_address_resolutions_hourly", @@ -275,7 +275,12 @@ export const pipes = [ { name: "product_events_mv", content: - "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + "DESCRIPTION >\n Populates product_events from session_events navigation and custom rows, with domain(Url)/path(Url) pre-extracted, the event name normalized and the SDK-stamped identity copied through.\n\nNODE product_events_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'browser' AS Source,\n SessionId,\n Seq,\n VisitorId,\n UserId,\n GroupId,\n Type AS Kind,\n if(Type = 'navigation', '$pageview', Message) AS EventName,\n domain(Url) AS Host,\n path(Url) AS PagePath,\n Url,\n '' AS ServiceName,\n Attributes,\n '' AS TraceId,\n '' AS SpanId\n FROM session_events\n WHERE Type IN ('navigation', 'custom')\n\nTYPE MATERIALIZED\nDATASOURCE product_events", + }, + { + name: "product_events_traces_mv", + content: + "DESCRIPTION >\n Populates product_events from spans carrying the maple.product_event.name attribute, projecting the span's identity, prop.* attributes, service and TraceId/SpanId so the event links back to the trace that produced it.\n\nNODE product_events_traces_mv_node\nSQL >\n SELECT\n OrgId,\n Timestamp,\n 'trace' AS Source,\n SpanAttributes['session.id'] AS SessionId,\n 0 AS Seq,\n SpanAttributes['maple.product_event.visitor_id'] AS VisitorId,\n SpanAttributes['maple.product_event.user_id'] AS UserId,\n SpanAttributes['maple.product_event.group_id'] AS GroupId,\n 'custom' AS Kind,\n SpanAttributes['maple.product_event.name'] AS EventName,\n domain(SpanAttributes['maple.product_event.url']) AS Host,\n path(SpanAttributes['maple.product_event.url']) AS PagePath,\n SpanAttributes['maple.product_event.url'] AS Url,\n ServiceName,\n mapUpdate(\n CAST(\n mapFilter(\n (k, v) -> NOT startsWith(k, 'maple.product_event.')\n AND (\n NOT has(mapKeys(SpanAttributes), 'maple.product_event.include')\n OR has(\n arrayMap(\n key -> trimBoth(key),\n splitByChar(',', SpanAttributes['maple.product_event.include'])\n ),\n k\n )\n ),\n SpanAttributes\n ),\n 'Map(String, String)'\n ),\n mapApply(\n (k, v) -> (substring(k, 26), v),\n mapFilter((k, v) -> startsWith(k, 'maple.product_event.prop.'), SpanAttributes)\n )\n ) AS Attributes,\n TraceId,\n SpanId\n FROM traces\n WHERE SpanAttributes['maple.product_event.name'] != ''\n\nTYPE MATERIALIZED\nDATASOURCE product_events", }, { name: "service_external_edges_hourly_mv", diff --git a/packages/domain/src/http/query-engine.ts b/packages/domain/src/http/query-engine.ts index 5f30d3c7e..cf5091067 100644 --- a/packages/domain/src/http/query-engine.ts +++ b/packages/domain/src/http/query-engine.ts @@ -42,6 +42,31 @@ const BucketSeconds = Schema.Number.check(Schema.isInt(), Schema.isGreaterThan(0 }), ) +/** + * A `LIMIT` a client is allowed to ask for. + * + * The builder INLINES a limit into the SQL text (`raw(String(Math.round(v)))`) + * rather than binding it, so an unchecked `Schema.Number` here is the + * `bucket_seconds: 1.5` mistake in a second costume: `limit: -1` and + * `limit: 1e21` (which stringifies as `1e+21`) both reach ClickHouse as a + * syntax error and surface as a 500, and `limit: 1e9` is an unbounded scan + * bounded only by the profile's own settings. + * + * The ceiling is the part that has to be here rather than in the caller: the + * web client's own input schema checks positivity but has no upper bound, and + * the internal API is reachable by any authenticated client regardless. + */ +const RowLimit = Schema.Number.check( + Schema.isInt(), + Schema.isGreaterThan(0), + Schema.isLessThanOrEqualTo(1000), +).pipe( + Schema.annotate({ + identifier: "RowLimit", + description: "Maximum rows to return: a whole number between 1 and 1000.", + }), +) + // Dedicated endpoint schemas /** Shared primitives for filtered list/facet endpoints. */ @@ -1742,6 +1767,72 @@ export class ProductEventNamesResponse extends Schema.Class( + "ProductEventsForTraceRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + traceId: TraceId, + /** Default 50, max 1000. */ + limit: Schema.optional(RowLimit), +}) {} + +export class ProductEventsForTraceResponse extends Schema.Class( + "ProductEventsForTraceResponse", +)({ + data: Schema.Array( + Schema.Struct({ + timestamp: Schema.String, + eventName: Schema.String, + /** The annotated span within the trace. */ + spanId: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + groupId: Schema.String, + visitorId: Schema.String, + sessionId: Schema.String, + /** The event's `maple.product_event.prop.*` attributes, prefix stripped. */ + attributes: Schema.Record(Schema.String, Schema.String), + }), + ), +}) {} + +/** Recent traces behind one event name — the analytics side of the same link. */ +export class ProductEventTraceSamplesRequest extends Schema.Class( + "ProductEventTraceSamplesRequest", +)({ + startTime: TinybirdDateTime, + endTime: TinybirdDateTime, + eventName: Schema.String, + /** Default 20, max 1000. */ + limit: Schema.optional(RowLimit), +}) {} + +export class ProductEventTraceSamplesResponse extends Schema.Class( + "ProductEventTraceSamplesResponse", +)({ + data: Schema.Array( + Schema.Struct({ + traceId: Schema.String, + spanId: Schema.String, + timestamp: Schema.String, + serviceName: Schema.String, + userId: Schema.String, + visitorId: Schema.String, + }), + ), +}) {} + export class PodFacetsRequest extends Schema.Class("PodFacetsRequest")({ startTime: TinybirdDateTime, endTime: TinybirdDateTime, @@ -2699,6 +2790,20 @@ export class QueryEngineApiGroup extends HttpApiGroup.make("queryEngine") error: queryEngineEndpointErrors, }), ) + .add( + HttpApiEndpoint.post("productEventsForTrace", "/product-events-for-trace", { + payload: ProductEventsForTraceRequest, + success: ProductEventsForTraceResponse, + error: queryEngineEndpointErrors, + }), + ) + .add( + HttpApiEndpoint.post("productEventTraceSamples", "/product-event-trace-samples", { + payload: ProductEventTraceSamplesRequest, + success: ProductEventTraceSamplesResponse, + error: queryEngineEndpointErrors, + }), + ) .add( HttpApiEndpoint.post("executeRawSql", "/execute-raw-sql", { payload: RawSqlExecuteRequest, diff --git a/packages/domain/src/tinybird/datasources.ts b/packages/domain/src/tinybird/datasources.ts index afb7962fc..9ea87c0bb 100644 --- a/packages/domain/src/tinybird/datasources.ts +++ b/packages/domain/src/tinybird/datasources.ts @@ -2247,6 +2247,30 @@ export const productEvents = defineDatasource("product_events", { Attributes: column(t.map(t.string(), t.string()).defaultExpr("map()"), { jsonPath: "$.attributes", }), + /** + * The trace this event was derived from — set on `Source = 'trace'` rows, + * `''` on every other source. This column IS the link, in both directions: + * the trace view lists the product events a trace produced, and a funnel + * row hands back the exact trace that performed the step. + * + * A real column rather than an `Attributes` key because both directions + * filter on it, and a `Map` lookup on this table reads the whole map per + * row — the cost `product_events` was split out of `session_events` to + * avoid. Last in the schema because `ALTER TABLE … ADD COLUMN` appends. + * + * NO `jsonPath`, deliberately, and load-bearing: these two are written + * only by `product_events_traces_mv` and its backfill, never by an + * ingested NDJSON line. `scripts/generate-clickhouse-insert-mappings.ts` + * skips columns whose path is null, so the Rust gateway's + * `INSERT INTO product_events (…)` does not name them — which is what lets + * migration 0024 stay `requiredForIngest: false`. Give them a path and the + * gateway starts naming columns that a BYO cluster stamped below 24 does + * not have, and every `/v1/events` batch for those orgs is rejected and + * dropped. Same shape as `service_usage`, whose columns are MV-only too. + */ + TraceId: t.string().default(""), + /** The annotated span within {@link TraceId}. `''` on non-trace rows. */ + SpanId: t.string().default(""), }, engine: engine.mergeTree({ partitionKey: "toDate(Timestamp)", @@ -2272,6 +2296,17 @@ export const productEvents = defineDatasource("product_events", { type: "bloom_filter", granularity: 4, }, + { + // "Which product events did this trace produce?" — the trace view asks + // it by id with no other predicate, so without this the lookup scans + // every partition in the retained window. Near-unique values, and the + // column is `''` on the overwhelming majority of rows, so a bloom + // filter both prunes hard and stays cheap. + name: "idx_trace_id", + expr: "TraceId", + type: "bloom_filter", + granularity: 4, + }, ], }) diff --git a/packages/domain/src/tinybird/materializations.ts b/packages/domain/src/tinybird/materializations.ts index 279aa13ee..5250585d4 100644 --- a/packages/domain/src/tinybird/materializations.ts +++ b/packages/domain/src/tinybird/materializations.ts @@ -47,6 +47,10 @@ import { DB_SYSTEM_ATTR_SQL, } from "./db-query-shape-sql" import { MAPLE_AI_SESSION_ID_ATTR, MAPLE_AI_VENDOR_ID_ATTR } from "../gen-ai" +import { + PRODUCT_EVENTS_TRACE_FILTER, + PRODUCT_EVENTS_TRACE_PROJECTION_SQL, +} from "./product-event-attributes" import { DEPLOYMENT_ENV_SQL, MESSAGING_DESTINATION_SQL } from "./semconv-renames" import { NORMALIZED_SPAN_NAME_SQL } from "./span-display-name" @@ -1619,7 +1623,9 @@ export const productEventsMv = defineMaterializedView("product_events_mv", { path(Url) AS PagePath, Url, '' AS ServiceName, - Attributes + Attributes, + '' AS TraceId, + '' AS SpanId FROM session_events WHERE Type IN ('navigation', 'custom') `, @@ -1627,6 +1633,41 @@ export const productEventsMv = defineMaterializedView("product_events_mv", { ], }) +/** + * Populates `product_events` from spans the customer annotated in their own + * code — `maple.product_event.name` on any span they already emit. + * + * This is the third feed into the table (browser via the MV above, server and + * mobile via `POST /v1/events`), and the only one that carries `TraceId`, which + * is what makes a product event and the trace that produced it navigable from + * either side. + * + * The predicate is one map lookup per incoming span, on the same block every + * other `traces` MV already fires on. It buys no skip-index help — an MV sees + * the insert block, not the table — so it is a real per-span cost at ingest, and + * a deliberately small one: a single `Map` value read on a map the block has + * just materialized anyway. + * + * Column order must match the `product_events` SCHEMA order — enforced by + * `materialized-projection-order.test.ts`. + */ +export const productEventsTracesMv = defineMaterializedView("product_events_traces_mv", { + description: + "Populates product_events from spans carrying the maple.product_event.name attribute, projecting the span's identity, prop.* attributes, service and TraceId/SpanId so the event links back to the trace that produced it.", + datasource: productEvents, + nodes: [ + node({ + name: "product_events_traces_mv_node", + sql: ` + SELECT + ${PRODUCT_EVENTS_TRACE_PROJECTION_SQL} + FROM traces + WHERE ${PRODUCT_EVENTS_TRACE_FILTER} + `, + }), + ], +}) + /** * Populates `identity_links` from `session_replays` rows that carry both a * visitor and a user id. diff --git a/packages/domain/src/tinybird/product-event-attributes.test.ts b/packages/domain/src/tinybird/product-event-attributes.test.ts new file mode 100644 index 000000000..a9f89a42a --- /dev/null +++ b/packages/domain/src/tinybird/product-event-attributes.test.ts @@ -0,0 +1,137 @@ +import { describe, expect, it } from "vitest" +import { + PRODUCT_EVENTS_TRACE_FILTER, + PRODUCT_EVENTS_TRACE_PROJECTION_SQL, + PRODUCT_EVENT_ATTRIBUTE_NAMESPACE, + PRODUCT_EVENT_INCLUDE_KEY, + PRODUCT_EVENT_NAME_KEY, + PRODUCT_EVENT_PROP_PREFIX, + PRODUCT_EVENT_SOURCE_TRACE, +} from "./product-event-attributes" +import { productEvents } from "./datasources" + +describe("product event span attributes", () => { + it("namespaces every key under maple.", () => { + // The `maple.*` vendor namespace is the convention every custom attribute + // in the repo follows, and it is what keeps these from colliding with an + // OTel semconv key that might later mean something else. + expect(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE.startsWith("maple.")).toBe(true) + expect(PRODUCT_EVENT_NAME_KEY.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + expect(PRODUCT_EVENT_INCLUDE_KEY.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + expect(PRODUCT_EVENT_PROP_PREFIX.startsWith(PRODUCT_EVENT_ATTRIBUTE_NAMESPACE)).toBe(true) + }) + + it("casts the base map to the target key type", () => { + // Not cosmetic: `SpanAttributes` is keyed `LowCardinality(String)` and + // `product_events.Attributes` is keyed plain `String`. Without the CAST the + // MV's SELECT has a different type from the column it writes, and + // `mapUpdate` has two differently-keyed maps to merge. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain("'Map(String, String)'") + }) + + it("switches the allow-list on key PRESENCE, not on a non-empty value", () => { + // The whole overwrite idiom rests on this. `include: ''` has to mean "no + // span attributes", so the test must be `mapKeys` containment — a `!= ''` + // here would silently turn the documented overwrite into the default + // copy-everything, which is the exact opposite of what the caller asked for. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `has(mapKeys(SpanAttributes), '${PRODUCT_EVENT_INCLUDE_KEY}')`, + ) + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).not.toContain( + `SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}'] != ''`, + ) + }) + + it("trims the allow-list entries", () => { + // `"plan, seats"` is what a human writes. Without the trim the second key + // never matches and the prop silently vanishes from every event. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain("trimBoth") + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `splitByChar(',', SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}'])`, + ) + }) + + it("strips exactly the prop prefix and nothing more", () => { + // ClickHouse `substring` is 1-indexed, so the offset is length + 1. Off by + // one in either direction is silent: one short leaves a leading `.` on + // every prop key, one long eats the first character of every prop NAME — + // `plan` becomes `lan` — and both produce a valid map no breakdown can + // group on. + const offset = /substring\(k, (\d+)\)/.exec(PRODUCT_EVENTS_TRACE_PROJECTION_SQL)?.[1] + expect(offset).toBe(String(PRODUCT_EVENT_PROP_PREFIX.length + 1)) + }) + + it("merges props over the base so an explicit prop wins a collision", () => { + // Argument order in `mapUpdate(base, props)` IS the override rule. Swapped, + // a team overriding a derived value would find their override discarded + // exactly when the key they wanted to correct was already present. + const merge = /mapUpdate\(\s*CAST\(/.exec(PRODUCT_EVENTS_TRACE_PROJECTION_SQL) + expect(merge, "props must be the SECOND mapUpdate argument").not.toBeNull() + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL.indexOf("mapApply")).toBeGreaterThan( + PRODUCT_EVENTS_TRACE_PROJECTION_SQL.indexOf("mapUpdate"), + ) + }) + + it("keeps the control namespace out of the copied attributes", () => { + // Once `prop.*` exists, leaving the namespace in means every explicit prop + // appears twice — as `plan` from the merge and as + // `maple.product_event.prop.plan` from the base — and name/user_id + // duplicate columns this same SELECT already promotes. + expect(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).toContain( + `NOT startsWith(k, '${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}')`, + ) + }) + + it("filters on a non-empty name rather than key presence", () => { + // `mapContains` would admit a span whose attribute is set to '' and mint a + // nameless event — a row no funnel can step on and no reader can attribute. + expect(PRODUCT_EVENTS_TRACE_FILTER).toBe(`SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] != ''`) + }) + + it("projects the product_events columns in schema order", () => { + // The MV body is checked structurally by materialized-projection-order, + // which reads the generated manifest. This asserts the same thing about the + // shared constant itself, so a bad edit fails before the manifest is + // regenerated rather than after. + // + // Split on TOP-LEVEL commas only. A line-wise scan was enough while every + // projected column was one line; `Attributes` is now a nested expression + // whose inner `mapFilter`/`arrayMap` lambdas contain both commas and bare + // identifiers, and a naive scan reads `k` and `SpanAttributes` as columns. + const topLevelParts = (input: string): ReadonlyArray => { + const parts: string[] = [] + let start = 0 + let depth = 0 + let inString = false + for (let index = 0; index < input.length; index++) { + const char = input[index] + if (char === "'" && input[index - 1] !== "\\") { + inString = !inString + continue + } + if (inString) continue + if (char === "(") depth++ + if (char === ")") depth-- + if (char === "," && depth === 0) { + parts.push(input.slice(start, index)) + start = index + 1 + } + } + parts.push(input.slice(start)) + return parts.map((part) => part.trim()).filter((part) => part.length > 0) + } + + const projected = topLevelParts(PRODUCT_EVENTS_TRACE_PROJECTION_SQL).map((part) => { + const aliased = /\bAS ([A-Za-z][A-Za-z0-9_]*)$/.exec(part) + return aliased ? aliased[1]! : part + }) + expect(projected).toEqual(Object.keys(productEvents._schema)) + }) + + it("carries a Source distinct from the other three feeds", () => { + // `Source` is the provenance column every reader branches on to tell an + // annotated span from a browser page view or a POST /v1/events row. + expect(PRODUCT_EVENT_SOURCE_TRACE).toBe("trace") + expect(["browser", "server", "mobile"]).not.toContain(PRODUCT_EVENT_SOURCE_TRACE) + }) +}) diff --git a/packages/domain/src/tinybird/product-event-attributes.ts b/packages/domain/src/tinybird/product-event-attributes.ts new file mode 100644 index 000000000..4e8e1fb1a --- /dev/null +++ b/packages/domain/src/tinybird/product-event-attributes.ts @@ -0,0 +1,191 @@ +/** + * The span-attribute contract that turns an instrumented span into a product + * event — "annotate in code, not in the UI". + * + * A customer marks a span they already emit: + * + * ```ts + * span.setAttributes({ + * "maple.product_event.name": "checkout_completed", + * "maple.product_event.user_id": user.id, + * }) + * ``` + * + * The span's own attributes come along whole by default — every key it already + * carries becomes an event property, with no opt-in list and no prefix to + * remember. The span IS the event, so anything worth putting on the span is + * worth breaking the funnel down by. + * + * Two optional controls narrow or replace that default, both themselves span + * attributes, because a materialized view is static SQL per cluster and has no + * per-org config to read: + * + * ```ts + * "maple.product_event.include": "plan,seats" // ONLY these span keys + * "maple.product_event.prop.plan": "pro" // explicit prop, wins ties + * "maple.product_event.include": "" // and together: full overwrite + * ``` + * + * Three tiers out of one mechanism rather than three modes to choose between: + * `include` narrows the base map, `prop.*` merges over it, and an EMPTY + * `include` narrows the base to nothing so the props are all that survive. There + * is no separate "replace" flag to get wrong. + * + * and `product_events_traces_mv` projects it into `product_events` with + * `Source = 'trace'` and the span's `TraceId`/`SpanId` carried through, so the + * event steps in a funnel like any `track()` call AND links back to the trace it + * came from. Nothing is written by hand and nothing is stored twice: the span is + * the record, the product event is its projection. + * + * Why an attribute rather than a UI action: a product event has to be emitted by + * the code path that actually performed the thing, at the moment it performed + * it. A human clicking "this trace was a signup" a day later marks ONE sampled + * trace, cannot be replayed over history, and puts a mutable, user-authored row + * into an append-only fact table. An attribute marks every trace the path + * produces, forever, and is reviewable in the customer's own diff. + * + * These keys are read in exactly two places, which must agree byte for byte: + * `productEventsTracesMv` in `materializations.ts` (managed orgs) and the frozen + * MV body in ClickHouse migration 0024 (BYO clusters). + */ + +/** Vendor namespace prefix. Every key below starts with it. */ +export const PRODUCT_EVENT_ATTRIBUTE_NAMESPACE = "maple.product_event." + +/** + * Required. Its presence — a non-empty value — is the whole predicate: a span + * carrying it becomes a product event, a span without it is ignored. The value + * becomes `EventName`, i.e. the funnel step key. + */ +export const PRODUCT_EVENT_NAME_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}name` + +/** Optional identity. Absent keys project to `''`, which means "unidentified". */ +export const PRODUCT_EVENT_USER_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}user_id` +export const PRODUCT_EVENT_GROUP_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}group_id` +export const PRODUCT_EVENT_VISITOR_ID_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}visitor_id` + +/** + * Optional page context, for events that belong to a URL (a server-rendered + * checkout, a webhook that knows the page it came from). + */ +export const PRODUCT_EVENT_URL_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}url` + +/** + * Optional allow-list: a comma-separated list of span attribute keys, and only + * those are copied into `Attributes`. Whitespace around each entry is trimmed, + * so `"plan, seats"` and `"plan,seats"` mean the same thing. + * + * PRESENCE is what switches the behaviour, not the value — which is why the + * projection tests it with `mapContains` and not `!= ''`. Present and empty + * therefore means "no span attributes at all", and that is the documented way to + * OVERWRITE rather than narrow: set it empty and supply {@link + * PRODUCT_EVENT_PROP_PREFIX} props, and the event carries exactly those. + */ +export const PRODUCT_EVENT_INCLUDE_KEY = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}include` + +/** + * Prefix for explicit props. `maple.product_event.prop.plan` lands in + * `Attributes` as `plan`, MERGED OVER whatever the base map produced and winning + * on a key collision — so a team can override one derived value without having + * to enumerate everything else with {@link PRODUCT_EVENT_INCLUDE_KEY}. + */ +export const PRODUCT_EVENT_PROP_PREFIX = `${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}prop.` + +/** + * OTel's own session key, read as a fallback so a browser-originated trace can + * stitch to the same session its `session_events` rows carry. + */ +export const PRODUCT_EVENT_SESSION_ID_KEY = "session.id" + +/** + * `Source` value for a trace-derived row. Joins `browser` (session_events MV), + * `server` and `mobile` (`POST /v1/events`). + */ +export const PRODUCT_EVENT_SOURCE_TRACE = "trace" + +/** + * The `product_events` projection of one annotated span, shared byte-for-byte + * between the managed materialized view and the BYO-ClickHouse migration's + * frozen copy — as with 0021's browser projection, two copies of this SELECT is + * two chances for the same span to be counted two different ways. + * + * Column order matches the `product_events` SCHEMA order, `TraceId`/`SpanId` + * last because that is where `ALTER TABLE … ADD COLUMN` puts them. + * + * `Kind = 'custom'` deliberately: `Kind` is the page-view predicate every reader + * already branches on, and a trace-derived event is a tracked event, not a page + * view. Provenance lives in `Source`, which is what the trace link reads. + * + * `Seq = 0` like every other direct row; ordering inside a millisecond comes + * from the span's own nanosecond `Timestamp`. + * + * `Attributes` is built in two halves and merged, `mapUpdate` letting the props + * win a collision: + * + * base = span attributes, minus the `maple.product_event.*` control + * namespace, optionally narrowed to `include`'s allow-list + * props = `maple.product_event.prop.*`, prefix stripped + * + * The default — neither control set — is the whole span map, which is the + * expensive choice and the deliberate one: a server span carries its full + * HTTP/DB semconv surface, and `product_events` keeps 365 days against raw + * `traces`' 30, so an annotated span's attributes outlive the span by a factor + * of twelve. What it buys is that nothing has to be declared to get started, and + * `include` is the lever for a team that has measured the cost and wants it back. + * + * The control namespace IS stripped from the base, unlike an earlier cut of + * this projection. Once `prop.*` exists, leaving it in means every explicit prop + * appears twice — once as `plan` from the merge and once as + * `maple.product_event.prop.plan` from the base — and `name`/`user_id` duplicate + * columns this same SELECT already promotes. + * + * The whole expression only evaluates for rows that pass the WHERE below, i.e. + * annotated spans, so its cost is paid per product event rather than per span. + * The predicate itself stays a single map lookup. + */ +export const PRODUCT_EVENTS_TRACE_PROJECTION_SQL = `OrgId, + Timestamp, + '${PRODUCT_EVENT_SOURCE_TRACE}' AS Source, + SpanAttributes['${PRODUCT_EVENT_SESSION_ID_KEY}'] AS SessionId, + 0 AS Seq, + SpanAttributes['${PRODUCT_EVENT_VISITOR_ID_KEY}'] AS VisitorId, + SpanAttributes['${PRODUCT_EVENT_USER_ID_KEY}'] AS UserId, + SpanAttributes['${PRODUCT_EVENT_GROUP_ID_KEY}'] AS GroupId, + 'custom' AS Kind, + SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] AS EventName, + domain(SpanAttributes['${PRODUCT_EVENT_URL_KEY}']) AS Host, + path(SpanAttributes['${PRODUCT_EVENT_URL_KEY}']) AS PagePath, + SpanAttributes['${PRODUCT_EVENT_URL_KEY}'] AS Url, + ServiceName, + mapUpdate( + CAST( + mapFilter( + (k, v) -> NOT startsWith(k, '${PRODUCT_EVENT_ATTRIBUTE_NAMESPACE}') + AND ( + NOT has(mapKeys(SpanAttributes), '${PRODUCT_EVENT_INCLUDE_KEY}') + OR has( + arrayMap( + key -> trimBoth(key), + splitByChar(',', SpanAttributes['${PRODUCT_EVENT_INCLUDE_KEY}']) + ), + k + ) + ), + SpanAttributes + ), + 'Map(String, String)' + ), + mapApply( + (k, v) -> (substring(k, ${PRODUCT_EVENT_PROP_PREFIX.length + 1}), v), + mapFilter((k, v) -> startsWith(k, '${PRODUCT_EVENT_PROP_PREFIX}'), SpanAttributes) + ) + ) AS Attributes, + TraceId, + SpanId` + +/** + * The predicate, also shared. `!= ''` rather than `mapContains`: a key present + * with an empty value would otherwise mint an event with no name, which is a row + * no funnel can step on and no reader can attribute. + */ +export const PRODUCT_EVENTS_TRACE_FILTER = `SpanAttributes['${PRODUCT_EVENT_NAME_KEY}'] != ''` diff --git a/packages/query-engine/src/__sql_baseline__/catalog.sql b/packages/query-engine/src/__sql_baseline__/catalog.sql index 50e0b8cfa..4df230039 100644 --- a/packages/query-engine/src/__sql_baseline__/catalog.sql +++ b/packages/query-engine/src/__sql_baseline__/catalog.sql @@ -1103,6 +1103,26 @@ SELECT LIMIT 100 FORMAT JSON +-- builder:product-events:productEventsForTraceQuery:default [d151e174] +SELECT + Timestamp AS timestamp, + EventName AS eventName, + SpanId AS spanId, + ServiceName AS serviceName, + UserId AS userId, + GroupId AS groupId, + VisitorId AS visitorId, + SessionId AS sessionId, + Attributes AS attributes + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND TraceId = '4bf92f3577b34da6a3ce929d0e0e4736' + ORDER BY timestamp ASC, spanId ASC + LIMIT 50 + FORMAT JSON + -- builder:product-events:productEventsFunnelBreakdownQuery:attribute-session-step [ac39fa69] SELECT group AS group, @@ -1467,6 +1487,24 @@ SELECT ORDER BY step ASC FORMAT JSON +-- builder:product-events:productEventTraceSamplesQuery:default [ee1608d5] +SELECT + TraceId AS traceId, + SpanId AS spanId, + Timestamp AS timestamp, + ServiceName AS serviceName, + UserId AS userId, + VisitorId AS visitorId + FROM product_events + WHERE OrgId = 'org_sql_catalog' + AND Timestamp >= '2026-01-01 10:30:00' + AND Timestamp <= '2026-01-03 14:15:00' + AND EventName = 'checkout_completed' + AND TraceId != '' + ORDER BY timestamp DESC + LIMIT 20 + FORMAT JSON + -- builder:service-endpoints:serviceEndpointsSummaryQuery:default [3e379104] SELECT bSpanName AS spanName, diff --git a/packages/query-engine/src/ch/builder-fixtures.ts b/packages/query-engine/src/ch/builder-fixtures.ts index 69d8f7adf..7d76ebf59 100644 --- a/packages/query-engine/src/ch/builder-fixtures.ts +++ b/packages/query-engine/src/ch/builder-fixtures.ts @@ -311,6 +311,29 @@ const productEventsFixtures: ReadonlyArray = [ window, ), }, + // The two directions of the trace ↔ product-event link. Both are + // single-predicate lookups on `TraceId`, so what the sweep is watching for is + // that neither grows a `Map` read or loses its `OrgId`/time bounds. + { + module: "product-events", + name: "productEventsForTraceQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.productEventsForTraceQuery({ limit: 50 }), { + ...window, + traceId: "4bf92f3577b34da6a3ce929d0e0e4736", + }), + }, + { + module: "product-events", + name: "productEventTraceSamplesQuery", + label: "default", + compile: () => + CH.compileUnsafe(CH.productEventTraceSamplesQuery({ limit: 20 }), { + ...window, + eventName: "checkout_completed", + }), + }, ] export const builderFixtures: ReadonlyArray = [ diff --git a/packages/query-engine/src/ch/index.ts b/packages/query-engine/src/ch/index.ts index 04b1e4433..6352392f3 100644 --- a/packages/query-engine/src/ch/index.ts +++ b/packages/query-engine/src/ch/index.ts @@ -189,6 +189,8 @@ export { productEventsFunnelBreakdownRowSchema, productEventNamesQuery, productEventNamesRowSchema, + productEventsForTraceQuery, + productEventTraceSamplesQuery, ProductEventsFunnelError, FUNNEL_MAX_STEPS, FUNNEL_BREAKDOWN_MAX_GROUPS, @@ -196,6 +198,10 @@ export { type FunnelKeyBy, type FunnelSessionDimension, type FunnelBreakdownBy, + type ProductEventsForTraceOpts, + type ProductEventForTraceOutput, + type ProductEventTraceSamplesOpts, + type ProductEventTraceSampleOutput, type ProductEventsFunnelOpts, type ProductEventsFunnelOutput, type ProductEventsFunnelBreakdownOpts, diff --git a/packages/query-engine/src/ch/queries/product-events.ts b/packages/query-engine/src/ch/queries/product-events.ts index eceedfcea..18480fc26 100644 --- a/packages/query-engine/src/ch/queries/product-events.ts +++ b/packages/query-engine/src/ch/queries/product-events.ts @@ -694,3 +694,134 @@ export function productEventNamesQuery( .limit(limit) .format("JSON") } + +// Trace ↔ product event linking +// +// A span the customer annotated with `maple.product_event.name` lands in +// `product_events` as a `Source = 'trace'` row carrying its `TraceId`/`SpanId` +// (ClickHouse migration 0024 / `product_events_traces_mv`). That column is the +// link, and these two queries are the two directions of walking it: +// +// trace → its product events `productEventsForTraceQuery` +// event → the traces behind it `productEventTraceSamplesQuery` +// +// Both filter `TraceId` directly rather than reaching through `Attributes`, +// because both are single-predicate lookups that must prune on the table's +// `idx_trace_id` bloom filter — a `Map` read would take the whole map per row on +// the one table that exists to avoid exactly that. + +// No declared `rowSchema` on either query below. Every projected column is a +// plain String or the `Map(String, String)` the builder already derives as +// `Schema.Record(Schema.String, Schema.String)`, so a declared copy would be +// byte-identical to the derived one — which is exactly the class of schema the +// 2026-08 sweep deleted 22 of. A declared schema earns its place only when it +// NARROWS (a literal union, a null-collapsing transform) or when derivation is +// wrong (LEFT-JOIN nullability). Neither applies here, and an exported copy +// nothing passes to `compile` is a contract that silently drifts. + +export interface ProductEventForTraceOutput { + readonly timestamp: string + readonly eventName: string + readonly spanId: string + readonly serviceName: string + readonly userId: string + readonly groupId: string + readonly visitorId: string + readonly sessionId: string + readonly attributes: Record +} + +export interface ProductEventsForTraceOpts { + /** Default 50 — a single trace producing more than this is pathological. */ + readonly limit?: number +} + +/** + * The product events one trace produced, oldest first — the trace view's + * "this request performed these things" panel. + * + * Deliberately NOT time-bounded beyond the caller's window: a trace id is + * already near-unique, and `idx_trace_id` prunes on it, so adding a narrow time + * predicate would only risk missing an event whose span started either side of + * the window the caller happened to pass. Callers still bound the range, because + * `OrgId`/`Timestamp` lead the sorting key and partition pruning is what keeps + * this off every retained day. + * + * `Source` is not filtered: a row carrying a `TraceId` can only have come from + * the trace projection, so filtering it would be a second way to say the same + * thing and a second thing to keep in sync. + */ +export function productEventsForTraceQuery( + opts: ProductEventsForTraceOpts = {}, +): CHQuery { + return from(ProductEvents) + .select(($) => ({ + timestamp: $.Timestamp, + eventName: $.EventName, + spanId: $.SpanId, + serviceName: $.ServiceName, + userId: $.UserId, + groupId: $.GroupId, + visitorId: $.VisitorId, + sessionId: $.SessionId, + attributes: $.Attributes, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + $.TraceId.eq(param.string("traceId")), + ]) + .orderBy(["timestamp", "asc"], ["spanId", "asc"]) + .limit(opts.limit ?? 50) + .format("JSON") +} + +export interface ProductEventTraceSampleOutput { + readonly traceId: string + readonly spanId: string + readonly timestamp: string + readonly serviceName: string + readonly userId: string + readonly visitorId: string +} + +export interface ProductEventTraceSamplesOpts { + /** Default 20. */ + readonly limit?: number +} + +/** + * Recent traces behind one event name, newest first — the other direction: + * standing on a funnel step or an event in the analytics list, go look at what + * actually happened. + * + * Only annotated rows can answer this, so `TraceId != ''` is the filter rather + * than `Source = 'trace'`: it is the same set, and it is the predicate that + * makes the result USEFUL (a row with no trace id is not a sample of anything), + * so stating it that way keeps the query honest about why it excludes the + * browser and `/v1/events` rows. + */ +export function productEventTraceSamplesQuery( + opts: ProductEventTraceSamplesOpts = {}, +): CHQuery { + return from(ProductEvents) + .select(($) => ({ + traceId: $.TraceId, + spanId: $.SpanId, + timestamp: $.Timestamp, + serviceName: $.ServiceName, + userId: $.UserId, + visitorId: $.VisitorId, + })) + .where(($) => [ + $.OrgId.eq(param.string("orgId")), + $.Timestamp.gte(param.dateTimeString("startTime")), + $.Timestamp.lte(param.dateTimeString("endTime")), + $.EventName.eq(param.string("eventName")), + $.TraceId.neq(""), + ]) + .orderBy(["timestamp", "desc"]) + .limit(opts.limit ?? 20) + .format("JSON") +} diff --git a/packages/query-engine/src/ch/tables.ts b/packages/query-engine/src/ch/tables.ts index b647c2741..df821238e 100644 --- a/packages/query-engine/src/ch/tables.ts +++ b/packages/query-engine/src/ch/tables.ts @@ -761,6 +761,13 @@ export const ProductEvents = table("product_events", { ServiceName: T.string, // track() props. Attributes: T.map(T.string, T.string), + // The trace this event was derived from — non-empty only on Source='trace' + // rows, i.e. spans the customer annotated with `maple.product_event.name`. + // The link in both directions: trace view → its product events, funnel row → + // the trace that performed the step. + TraceId: T.string, + // The annotated span within TraceId. '' on every other source. + SpanId: T.string, }) // (VisitorId, UserId) pairs observed together on a session_replays row. diff --git a/packages/query-engine/src/registry/product-events.ts b/packages/query-engine/src/registry/product-events.ts index c11545d34..d11fb59f3 100644 --- a/packages/query-engine/src/registry/product-events.ts +++ b/packages/query-engine/src/registry/product-events.ts @@ -1,7 +1,9 @@ import type { ProductEventNamesRequest, + ProductEventsForTraceRequest, ProductEventsFunnelBreakdownRequest, ProductEventsFunnelRequest, + ProductEventTraceSamplesRequest, } from "@maple/domain/http" import * as CH from "../ch" import { timeRangeCache } from "../runtime/query-engine" @@ -81,3 +83,41 @@ export const productEventNames = defineQuery({ { orgId, startTime: payload.startTime, endTime: payload.endTime }, ), }) + +// The trace ↔ product-event link, both directions. `list` rather than +// `aggregation`: each reads a handful of rows off a single equality predicate, +// and paying the aggregation profile's settings for a bloom-filter point lookup +// is the wrong trade. +// +// A flat 60s rather than `timeRangeCache`, whose TTL and key-snap widen with the +// range: both of these are point lookups whose answer does not change with how +// much time the caller asked about, and a completed trace's events never change +// at all. 60s covers the burst of re-reads a panel does while someone clicks +// around one trace, and is short enough that an event annotated a minute ago +// shows up in the samples list. + +export const productEventsForTrace = defineQuery({ + id: "productEventsForTrace", + profile: "list", + cache: 60, + compile: (payload: ProductEventsForTraceRequest, orgId: string) => + CH.compile(CH.productEventsForTraceQuery({ limit: payload.limit ?? 50 }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + traceId: payload.traceId, + }), +}) + +export const productEventTraceSamples = defineQuery({ + id: "productEventTraceSamples", + profile: "list", + cache: 60, + compile: (payload: ProductEventTraceSamplesRequest, orgId: string) => + CH.compile(CH.productEventTraceSamplesQuery({ limit: payload.limit ?? 20 }), { + orgId, + startTime: payload.startTime, + endTime: payload.endTime, + eventName: payload.eventName, + }), +}) diff --git a/packages/query-engine/src/registry/queries.ts b/packages/query-engine/src/registry/queries.ts index a424554a9..8484f0884 100644 --- a/packages/query-engine/src/registry/queries.ts +++ b/packages/query-engine/src/registry/queries.ts @@ -56,7 +56,13 @@ import { makeTimeRangeCachePolicy, timeRangeCache } from "../runtime/query-engin import { defineQuery } from "./query-definition" export { logsCount, logsTimeseries } from "./logs" -export { productEventsFunnel, productEventsFunnelBreakdown, productEventNames } from "./product-events" +export { + productEventsFunnel, + productEventsFunnelBreakdown, + productEventNames, + productEventsForTrace, + productEventTraceSamples, +} from "./product-events" /** * Declarative compile, execution, and cache policy. Handlers retain response