From 35a5fdd82ef6038f45723f3c48c635eb5187975c Mon Sep 17 00:00:00 2001 From: CTWalk <100585900+CTWalk@users.noreply.github.com> Date: Mon, 10 Aug 2026 16:29:38 +0800 Subject: [PATCH] fix(evals): keep sampled-out items out of the experiment pass rate An item whose scorers were all sampled out has no evaluation evidence, but evaluateItemStatus() returns "passed" for it, so it counted in both the numerator and the denominator of summary.passRate. Lowering scorer sampling therefore pushed the pass rate toward 1, and a { type: "passRate" } criterion could pass on a run where nothing was scored. Exclude those items from the pass rate on both sides, matching the scorer aggregate, which already leaves skipped results out of its own denominator. A run with nothing evaluated now reports null. successCount, failureCount, errorCount, skippedCount and item status are unchanged. Developed with AI assistance. --- .../pass-rate-excludes-sampled-out-items.md | 5 ++ packages/evals/src/experiment/aggregator.ts | 14 +++- .../src/experiment/run-experiment.spec.ts | 78 +++++++++++++++++++ .../evaluation-docs/offline-evaluations.md | 4 +- 4 files changed, 98 insertions(+), 3 deletions(-) create mode 100644 .changeset/pass-rate-excludes-sampled-out-items.md diff --git a/.changeset/pass-rate-excludes-sampled-out-items.md b/.changeset/pass-rate-excludes-sampled-out-items.md new file mode 100644 index 000000000..3c44a998a --- /dev/null +++ b/.changeset/pass-rate-excludes-sampled-out-items.md @@ -0,0 +1,5 @@ +--- +"@voltagent/evals": patch +--- + +Fixed experiment `summary.passRate` counting items whose scorers were all sampled out as passes. Those items have no evaluation evidence, but `evaluateItemStatus()` returns `passed` for them, so lowering scorer sampling pushed the pass rate toward 1 and a `{ type: "passRate" }` criterion could pass on results that were never scored. Sampled-out items are now excluded from the pass rate, matching how skipped scorer results are already excluded from a scorer's own pass rate; a run with nothing evaluated reports `passRate: null`. `successCount`, `failureCount`, `errorCount`, and `skippedCount` are unchanged. diff --git a/packages/evals/src/experiment/aggregator.ts b/packages/evals/src/experiment/aggregator.ts index 74faf5e0d..684ed5fa0 100644 --- a/packages/evals/src/experiment/aggregator.ts +++ b/packages/evals/src/experiment/aggregator.ts @@ -32,6 +32,8 @@ export interface ExperimentAggregatorState { failureCount: number; errorCount: number; skippedCount: number; + evaluatedCount: number; + evaluatedSuccessCount: number; globalScoreSum: number; globalScoreCount: number; scorers: Map; @@ -47,6 +49,8 @@ export function createAggregatorState(totalHint?: number): ExperimentAggregatorS failureCount: 0, errorCount: 0, skippedCount: 0, + evaluatedCount: 0, + evaluatedSuccessCount: 0, globalScoreSum: 0, globalScoreCount: 0, scorers: new Map(), @@ -76,8 +80,16 @@ export function recordAggregatorResult( const scores = Object.values(item.scores); const allSkipped = scores.length > 0 && scores.every((score) => score.status === "skipped"); + // An item whose every scorer was skipped carries no evaluation evidence, so + // it stays out of the pass rate, the same way skipped scorer results stay + // out of a scorer's own pass rate. if (allSkipped) { state.skippedCount += 1; + } else { + state.evaluatedCount += 1; + if (item.status === "passed") { + state.evaluatedSuccessCount += 1; + } } for (const score of scores) { @@ -141,7 +153,7 @@ export function buildAggregatorSummary( errorCount: state.errorCount, skippedCount: state.skippedCount, meanScore: state.globalScoreCount > 0 ? state.globalScoreSum / state.globalScoreCount : null, - passRate: completedCount > 0 ? state.successCount / completedCount : null, + passRate: state.evaluatedCount > 0 ? state.evaluatedSuccessCount / state.evaluatedCount : null, startedAt: state.startedAt, scorers: buildScorerAggregates(state), criteria: [], diff --git a/packages/evals/src/experiment/run-experiment.spec.ts b/packages/evals/src/experiment/run-experiment.spec.ts index b69337b36..aef342339 100644 --- a/packages/evals/src/experiment/run-experiment.spec.ts +++ b/packages/evals/src/experiment/run-experiment.spec.ts @@ -9,6 +9,7 @@ const DATASET_ID = "dataset-integration"; const DATASET_VERSION_ID = "dataset-version-integration"; const DATASET_ITEM_1_ID = "11111111-1111-4111-8111-111111111111"; const DATASET_ITEM_2_ID = "22222222-2222-4222-8222-222222222222"; +const DATASET_ITEM_3_ID = "33333333-3333-4333-8333-333333333333"; function createDatasetItems(): ExperimentDatasetItem[] { return [ @@ -142,4 +143,81 @@ describe("runExperiment integration", () => { expect(result.summary.scorers.hede?.passRate).toBe(1); expect(result.summary.scorers).not.toHaveProperty("original-id"); }); + + it("keeps sampled-out items out of the pass rate", async () => { + const experiment = createExperiment({ + id: "run-sampled-out", + dataset: { + items: [ + { id: DATASET_ITEM_1_ID, input: "scored", expected: "scored" }, + { id: DATASET_ITEM_2_ID, input: "sampled-out", expected: "sampled-out" }, + { id: DATASET_ITEM_3_ID, input: "boom", expected: "boom" }, + ], + }, + runner: async ({ item }) => ({ output: item.input }), + scorers: [ + { + id: "sampled", + threshold: 0.5, + scorer: { + id: "sampled", + name: "sampled", + // One item is skipped, exactly as scorer sampling skips it, one + // scores below the threshold and one errors. + scorer: ({ payload }) => { + if (payload.input === "sampled-out") { + return { status: "skipped" }; + } + if (payload.input === "boom") { + return { status: "error", error: new Error("scorer failed") }; + } + return { status: "success", score: 0 }; + }, + }, + }, + ], + }); + + const result = await runExperiment(experiment); + + // Nothing passed: one item scored below its threshold and one errored, so + // the pass rate is 0 of the 2 evaluated items. Before this fix the + // sampled-out item counted as a success and reported passRate 0.33. + expect(result.summary.passRate).toBe(0); + expect(result.summary.skippedCount).toBe(1); + expect(result.summary.errorCount).toBe(1); + }); + + it("reports a null pass rate when every item is sampled out", async () => { + const experiment = createExperiment({ + id: "run-fully-sampled-out", + dataset: { + items: [{ id: DATASET_ITEM_1_ID, input: "hello", expected: "hello" }], + }, + runner: async ({ item }) => ({ output: item.input }), + scorers: [ + { + id: "sampled", + threshold: 0.5, + scorer: { + id: "sampled", + name: "sampled", + sampling: { type: "never" }, + scorer: () => ({ status: "success", score: 1 }), + }, + }, + ], + passCriteria: [ + { type: "passRate", min: 1 }, + { type: "passRate", min: 1, scorerId: "sampled" }, + ], + }); + + const result = await runExperiment(experiment); + + // Both spellings of the same criterion must agree: with no evaluated + // evidence there is no pass rate to meet, so neither can pass. + expect(result.summary.passRate).toBeNull(); + expect(result.summary.criteria.map((entry) => entry.passed)).toEqual([false, false]); + }); }); diff --git a/website/evaluation-docs/offline-evaluations.md b/website/evaluation-docs/offline-evaluations.md index 05aa9871e..83c62612e 100644 --- a/website/evaluation-docs/offline-evaluations.md +++ b/website/evaluation-docs/offline-evaluations.md @@ -164,7 +164,7 @@ While individual scorers determine if each item passes or fails, **pass criteria There are two types of criteria: - **`meanScore`**: Average score across all items must meet a minimum -- **`passRate`**: Percentage of passed items must meet a minimum +- **`passRate`**: Percentage of passed items must meet a minimum. Items whose scorers were all sampled out are not counted, since they carry no evaluation evidence ```ts import { createExperiment } from "@voltagent/evals"; @@ -785,7 +785,7 @@ interface ExperimentSummary { errorCount: number; // items with status "error" skippedCount: number; // items with status "skipped" meanScore?: number | null; - passRate?: number | null; + passRate?: number | null; // passed / evaluated items; null if nothing was evaluated startedAt: number; // Unix timestamp completedAt?: number; // Unix timestamp durationMs?: number;