diff --git a/.agents/AGENTS.md b/.agents/AGENTS.md index fa9a057efa..493044c0c8 100644 --- a/.agents/AGENTS.md +++ b/.agents/AGENTS.md @@ -114,6 +114,7 @@ These run `pnpm build` followed by `pnpm link-check` / `pnpm sitemap-check`. The - Use sentence case for user-facing headlines, section headings, and hero copy by default. Keep title case for short standalone navigation/UI labels where it reads more naturally (for example, paired nouns like "Questions & Answers" or conventional labels like "Get Started"). Always preserve proper nouns, acronyms, and official product names. - Route sales and Enterprise inquiries to the [sales form](/talk-to-us) instead of directing readers to `enterprise@langfuse.com`. - Add an `` to a feature's docs page when the feature is not available on every Langfuse plan or deployment type. Place it directly below the relevant heading: usually the H1, or an H2/H3 when availability applies only to that section. +- Use Fumadocs `` for object/field data models (name, type, description, default, required, deprecated) instead of a markdown table. It is registered in `mdx-components.tsx`; markdown/PDF export is handled in `lib/markdown-component-renderers.js`. Keep markdown tables for comparisons, region/path lists, and other non-typed layouts. Examples: Score and Dataset data-model pages. - Never reference internal ticket ids (`LFE-1234`, `LFINT-1234`) or Linear URLs in page content, commit messages, or PR descriptions. They mean nothing to readers of the public site or repo. Describe the change on its own terms; a ticket-prefixed branch name is the one place the identifier belongs. ### Changelog entries diff --git a/components/docs/type-table.tsx b/components/docs/type-table.tsx new file mode 100644 index 0000000000..b435cd791f --- /dev/null +++ b/components/docs/type-table.tsx @@ -0,0 +1,15 @@ +import { TypeTable as FumadocsTypeTable } from "fumadocs-ui/components/type-table"; +import type { ComponentProps } from "react"; +import { cn } from "@/lib/utils"; + +type TypeTableProps = ComponentProps; + +/** + * Fumadocs TypeTable styled to match Langfuse docs tables: rectangular, + * 1px structure border, no card radius or shadow. + */ +export function TypeTable({ className, ...props }: TypeTableProps) { + return ( + + ); +} diff --git a/content/docs/evaluation/experiments/data-model.mdx b/content/docs/evaluation/experiments/data-model.mdx index 9b16745870..5ee3d54556 100644 --- a/content/docs/evaluation/experiments/data-model.mdx +++ b/content/docs/evaluation/experiments/data-model.mdx @@ -59,39 +59,118 @@ direction LR #### Dataset object [#dataset-object] -| Attribute | Type | Required | Description | -| ------------------------- | ------ | -------- | ------------------------------------------- | -| `id` | string | Yes | Unique identifier for the dataset | -| `name` | string | Yes | Name of the dataset | -| `description` | string | No | Description of the dataset | -| `metadata` | object | No | Additional metadata for the dataset | -| `remoteExperimentUrl` | string | No | Webhook endpoint for triggering experiments | -| `remoteExperimentPayload` | object | No | Payload for triggering experiments | + #### DatasetItem object [#datasetitem-object] -| Attribute | Type | Required | Description | -| --------------------- | ------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | string | Yes | Unique identifier for the dataset item. Dataset items are upserted on their id. Id needs to be unique (project-level) and cannot be reused across datasets. | -| `datasetId` | string | Yes | ID of the dataset this item belongs to | -| `input` | object | No | Input data for the dataset item | -| `expectedOutput` | object | No | Expected output data for the dataset item | -| `metadata` | object | No | Additional metadata for the dataset item | -| `mediaReferences` | object[] | No | Resolved media references found in `input`, `expectedOutput`, and `metadata`. Included on SDK dataset fetches and API responses that include resolved dataset media. | -| `sourceTraceId` | string | No | ID of the source trace to link this dataset item to | -| `sourceObservationId` | string | No | ID of the source observation to link this dataset item to | -| `status` | DatasetStatus | No | Status of the dataset item. Defaults to ACTIVE for newly created items. Possible values: `ACTIVE`, `ARCHIVED` | + #### DatasetItemMediaReference object [#datasetitemmediareference-object] Dataset item media references point from a stored media token in `input`, `expectedOutput`, or `metadata` to a signed media download URL. -| Attribute | Type | Required | Description | -| ----------------- | ------ | -------------- | ------------------------------------------------------------------------------------------------------------------------------------------- | -| `field` | string | Yes | Field enum for the dataset item property containing the reference. One of `input`, `expected_output` (for `expectedOutput`), or `metadata`. | -| `referenceString` | string | Yes | Original Langfuse media reference string stored in the dataset item. | -| `jsonPath` | string | Yes | JSONPath of the string holding the reference inside the field, for example `$['image']`. | -| `media` | object | Yes (nullable) | Resolved media metadata. `null` if the referenced media does not exist or has not been uploaded successfully. | + The nested `media` object contains `mediaId`, `contentType`, `contentLength`, `url`, and `urlExpiry`. The `url` is a signed download URL and should be used before its expiration date. To refresh the signed URL, refetch the dataset. @@ -125,23 +204,64 @@ direction LR #### DatasetRun object [#datasetrun-object] -| Attribute | Type | Required | Description | -| ------------- | ------ | -------- | --------------------------------------- | -| `id` | string | Yes | Unique identifier for the dataset run | -| `name` | string | Yes | Name of the dataset run | -| `description` | string | No | Description of the dataset run | -| `metadata` | object | No | Additional metadata for the dataset run | -| `datasetId` | string | Yes | ID of the dataset this run belongs to | + #### DatasetRunItem object [#datasetrunitem-object] -| Attribute | Type | Required | Description | -| --------------- | ------ | -------- | ------------------------------------------ | -| `id` | string | Yes | Unique identifier for the dataset run item | -| `datasetRunId` | string | Yes | ID of the dataset run this item belongs to | -| `datasetItemId` | string | Yes | ID of the dataset item to link to this run | -| `traceId` | string | Yes | ID of the trace to link to this run | -| `observationId` | string | No | ID of the observation to link to this run | + Langfuse currently assumes that experiments do not contain repetitions: each dataset item appears once per experiment. Accordingly, reads surface at most one experiment item per dataset item within an experiment. Repetition support is tracked in [#5855](https://github.com/langfuse/langfuse/issues/5855). diff --git a/content/docs/evaluation/scores/data-model.mdx b/content/docs/evaluation/scores/data-model.mdx index 5cc7875310..6651de22b7 100644 --- a/content/docs/evaluation/scores/data-model.mdx +++ b/content/docs/evaluation/scores/data-model.mdx @@ -48,20 +48,70 @@ Scores have the following properties: ### Score object [#score-object] -| Attribute | Type | Required | Description | -| --------------- | ------ | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `id` | string | Yes | Unique identifier of the score. Auto-generated by SDKs. Optionally can also be used as an idempotency key to update scores. | -| `name` | string | Yes | Name of the score, e.g. user_feedback, hallucination_eval | -| `value` | number | No | Numeric value of the score. Always defined for numeric and boolean scores. Optional for categorical scores. Not used for text scores. | -| `stringValue` | string | No | String value of the score. Used for categorical, boolean (string equivalent), and text data types. Automatically set for categorical scores based on the config if the `configId` is provided. | -| `dataType` | string | No | Automatically set based on the config data type when the `configId` is provided. Otherwise can be defined manually as `NUMERIC`, `CATEGORICAL`, `BOOLEAN`, or `TEXT` | -| `source` | string | Yes | Automatically set based on the source of the score. Can be either `API`, `EVAL`, or `ANNOTATION` | -| `comment` | string | No | Evaluation comment, commonly used for user feedback, eval reasoning output or internal notes | -| `traceId` | string | No | Id of the trace the score relates to | -| `observationId` | string | No | Id of the observation (e.g. LLM call) the score relates to | -| `sessionId` | string | No | Id of the session the score relates to | -| `datasetRunId` | string | No | Id of the dataset run the score relates to | -| `configId` | string | No | Score config id to ensure that the score follows a specific schema. Can be defined in the Langfuse UI or via API. | + ### Common Use Cases [#common-use-cases] @@ -91,13 +141,49 @@ A score config includes: ### ScoreConfig object [#scoreconfig-object] -| Attribute | Type | Required | Description | -| ------------- | ------- | -------- | ------------------------------------------------------------------------------------- | -| `id` | string | Yes | Unique identifier of the score config. | -| `name` | string | Yes | Name of the score config, e.g. user_feedback, hallucination_eval | -| `dataType` | string | Yes | Can be either `NUMERIC`, `CATEGORICAL`, `BOOLEAN`, or `TEXT` | -| `isArchived` | boolean | No | Whether the score config is archived. Defaults to false | -| `minValue` | number | No | Sets minimum value for numerical scores. If not set, the minimum value defaults to -∞ | -| `maxValue` | number | No | Sets maximum value for numerical scores. If not set, the maximum value defaults to +∞ | -| `categories` | list | No | Defines categories for categorical scores. List of objects with label value pairs | -| `description` | string | No | Provides further description of the score configuration | + diff --git a/lib/markdown-component-renderers.js b/lib/markdown-component-renderers.js index 4010165d20..b2a8a43cb5 100644 --- a/lib/markdown-component-renderers.js +++ b/lib/markdown-component-renderers.js @@ -36,7 +36,9 @@ const CODING_AGENTS_SPEND_SUMMARY = function replaceComponentsWithMarkdown(fileContent) { const exportedResourceArrays = extractExportedResourceArrays(fileContent); return stripResourceArrayExports( - replaceCardGroupsWithMarkdown(replaceAcademyComponents(fileContent)), + replaceCardGroupsWithMarkdown( + replaceAcademyComponents(replaceTypeTablesWithMarkdown(fileContent)), + ), Object.keys(exportedResourceArrays), ) .replace( @@ -159,6 +161,110 @@ function replaceComponentsWithMarkdown(fileContent) { ); } +function replaceTypeTablesWithMarkdown(fileContent) { + return replaceSelfClosingComponents( + fileContent, + ["TypeTable"], + (_, attributes) => renderTypeTable(attributes), + ); +} + +/** + * Turns Fumadocs TypeTable JSX into a Markdown field table so .md / PDF / + * md-src output keeps the same attribute names, types, and descriptions. + * + * Type objects are authored as JS literals (`type={{ id: { type: "string" } }}`), + * so evaluate the expression instead of re-parsing field syntax by hand. + */ +function renderTypeTable(attributes) { + const typeExpr = extractJsxExpression(attributes, "type"); + if (!typeExpr) { + throw new Error("TypeTable requires a type object"); + } + + let type; + try { + type = new Function(`"use strict"; return (${typeExpr})`)(); + } catch (error) { + throw new Error( + `TypeTable type must be a JS object literal: ${error.message}`, + ); + } + + if (!type || typeof type !== "object" || Array.isArray(type)) { + throw new Error("TypeTable type must be an object literal"); + } + + const entries = Object.entries(type); + if (entries.length === 0) { + throw new Error("TypeTable type object is empty"); + } + + const includeDefault = entries.some(([, value]) => value?.default != null); + const includeDeprecated = entries.some(([, value]) => value?.deprecated); + + const headers = ["Attribute", "Type", "Required"]; + if (includeDefault) headers.push("Default"); + headers.push("Description"); + + const lines = [ + `| ${headers.join(" | ")} |`, + `| ${headers.map(() => "---").join(" | ")} |`, + ]; + + for (const [name, value] of entries) { + const field = value ?? {}; + const cells = [ + `\`${escapeTableCell(name)}\``, + field.type != null ? `\`${escapeTableCell(field.type)}\`` : "", + field.required ? "Yes" : "No", + ]; + if (includeDefault) { + cells.push( + field.default != null ? `\`${escapeTableCell(field.default)}\`` : "", + ); + } + + let description = field.description ?? ""; + if (includeDeprecated && field.deprecated) { + description = description ? `${description} Deprecated.` : "Deprecated."; + } + cells.push(escapeTableCell(description)); + lines.push(`| ${cells.join(" | ")} |`); + } + + return lines.join("\n"); +} + +/** Reads `prop={...}` and returns the expression source inside the braces. */ +function extractJsxExpression(attributes, propName) { + const start = attributes.search(new RegExp("\\b" + propName + "=\\{")); + if (start === -1) return null; + const open = attributes.indexOf("{", start + propName.length); + if (open === -1) return null; + + let depth = 0; + let quote = null; + for (let i = open; i < attributes.length; i++) { + const ch = attributes[i]; + if (quote) { + if (ch === "\\") i++; + else if (ch === quote) quote = null; + continue; + } + if (ch === '"' || ch === "'" || ch === "`") { + quote = ch; + continue; + } + if (ch === "{") depth++; + else if (ch === "}") { + depth--; + if (depth === 0) return attributes.slice(open + 1, i); + } + } + return null; +} + function renderSelfHostScaleMetrics() { return `Langfuse OSS and Enterprise use the same codebase as Langfuse Cloud. Langfuse processes **${formatObservationsPerMonth()} observations per month** and is trusted by **${FORTUNE_50_COMPANIES} of the Fortune 50**. Its Docker images have been pulled **${formatDockerPulls()} times**.`; } diff --git a/mdx-components.tsx b/mdx-components.tsx index 1a134ab756..8694c311ea 100644 --- a/mdx-components.tsx +++ b/mdx-components.tsx @@ -1,4 +1,5 @@ import defaultMdxComponents from "fumadocs-ui/mdx"; +import { TypeTable } from "@/components/docs/type-table"; import type { MDXComponents } from "mdx/types"; import React from "react"; import dynamic from "next/dynamic"; @@ -109,6 +110,7 @@ export function getMDXComponents(components?: MDXComponents): MDXComponents { Tabs: LangTabsWithTab, Tab: LangTab, table: Table, + TypeTable, Cards, Card, "Cards.Card": Card, diff --git a/src/overrides.css b/src/overrides.css index a54520caee..dfa300481e 100644 --- a/src/overrides.css +++ b/src/overrides.css @@ -1291,3 +1291,80 @@ inkeep-portal .inkeep-widget-vars { .prose blockquote p:last-of-type::after { content: none; } + +/* + * Fumadocs TypeTable — rectangular chrome matching markdown tables. + */ +.lf-type-table { + border-radius: 0 !important; + padding: 0 !important; + margin-top: 1.5rem; + margin-bottom: 1.5rem; + background-color: var(--surface-bg) !important; + color: var(--text-primary) !important; + border: 1px solid var(--line-structure) !important; + box-shadow: none !important; + overflow: visible !important; + font-size: 0.813rem; +} + +.lf-type-table > :first-child { + background-color: var(--surface-1); + border-bottom: 1px solid var(--line-structure); + color: var(--text-tertiary) !important; + font-size: 0.813rem; + font-weight: 500; + padding: 0.5rem 1rem !important; +} + +.lf-type-table > [data-state] { + border-radius: 0 !important; + border: none !important; + border-bottom: 1px solid var(--line-structure) !important; + box-shadow: none !important; + background-color: transparent !important; + margin: 0 !important; + overflow: hidden; +} + +.lf-type-table > [data-state]:last-child { + border-bottom: none !important; +} + +.lf-type-table > [data-state] > button { + padding: 0.5rem 1rem !important; + background-color: transparent; + border-radius: 0; +} + +.lf-type-table > [data-state] > button:hover { + background-color: var(--surface-1); +} + +.lf-type-table > [data-state] > button code { + color: var(--text-primary); + background: transparent; + border: none; + padding: 0; + font-size: 0.813rem; + font-weight: 500; +} + +.lf-type-table > [data-state] > button > span, +.lf-type-table > [data-state] > button > a { + color: var(--text-secondary); + font-size: 0.813rem; +} + +.lf-type-table > [data-state] > button svg { + color: var(--text-tertiary); + width: 0.875rem; + height: 0.875rem; +} + +.lf-type-table > [data-state] > [data-state] > div { + border-top-color: var(--line-structure) !important; + background-color: var(--surface-bg); + padding: 0.75rem 1rem !important; + color: var(--text-secondary); +}