From cb0f5af40b33398744babb7a2f1d90bc20355e68 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Tue, 15 Sep 2026 00:31:32 +0000 Subject: [PATCH 01/27] feat(events): server conformance for the merged Events design sketch, phase 1 Adds the requirement-traceability yaml for MCP Events plus the first two of five server scenarios, scoring against the design sketch that merged on main of modelcontextprotocol/experimental-ext-triggers-events on 2026-09-08. The SEP number is a placeholder. Events has no PR in modelcontextprotocol/modelcontextprotocol, and both traceability gates are numeric (the filename regex at src/traceability/index.ts:174, the check-id regex at line 41), so an unnumbered file is dropped from the manifest without an error. 9999 is far from the live range; the rename is mechanical and the yaml header names the rename as a merge blocker rather than a follow-up: a merge under 9999 would publish it as a real SEP. The reservation question is open with the WG. src/seps/sep-9999.yaml declares 131 checks and 30 excluded rows against 144 RFC 2119 keyword occurrences. Pure MAY and OPTIONAL sentences get no check. Client, host, receiver and SDK-guidance obligations are excluded rather than declared, so the denominator holds only what a server-side run can observe. events-discovery covers the capability, events/list, the descriptor fields and the error-code contract. events-poll covers poll delivery, the EventOccurrence shape and cursor lifecycle, driving a real two-poll quiet-period loop so cursor advancement is gradeable against a server with no traffic. Together they emit 45 rows; the other 86 report untested until the push and webhook scenarios land. The capability gate asks before it skips. An optional capability a server never declared is not a defect, but a server that answers events/list while declaring nothing has a surface no spec-following client would reach, and a SKIP would report that as a clean run. mcpkit is in exactly that state. Against examples/events/kitchen-sink at mcpkit main: discovery 8/11, poll 18/28. Six divergences, five of them server-side defects not previously tracked, one confirming the known nextPollSeconds drift. The yaml header lists each. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017kyEZdgh1zjdhURgdcZ4nL --- src/scenarios/index.ts | 18 +- src/scenarios/server/events/discovery.ts | 548 ++++++++ src/scenarios/server/events/helpers.ts | 391 ++++++ src/scenarios/server/events/negative.test.ts | 599 +++++++++ src/scenarios/server/events/poll.ts | 1208 ++++++++++++++++++ src/seps/sep-9999.yaml | 537 ++++++++ src/types.ts | 8 +- 7 files changed, 3307 insertions(+), 2 deletions(-) create mode 100644 src/scenarios/server/events/discovery.ts create mode 100644 src/scenarios/server/events/helpers.ts create mode 100644 src/scenarios/server/events/negative.test.ts create mode 100644 src/scenarios/server/events/poll.ts create mode 100644 src/seps/sep-9999.yaml diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 375d7010..cf43bda3 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -60,6 +60,8 @@ import { ResourcesNotFoundErrorScenario } from './server/resources'; +import { EventsDiscoveryScenario } from './server/events/discovery'; +import { EventsPollScenario } from './server/events/poll'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; @@ -165,7 +167,16 @@ const pendingClientScenariosList: ClientScenario[] = [ // `npm start -- server --scenario sep-2640-skills-* --url `. new SkillsDirectoryReadScenario(), new SkillsEnumerationScenario(), - new SkillsManifestScenario() + new SkillsManifestScenario(), + + // MCP Events. Pending because the everything-server does not implement the + // events capability; targeted runs point at an events-capable fixture via + // `npm start -- server --scenario events-* --url `. The suite + // scores against the merged design sketch in + // modelcontextprotocol/experimental-ext-triggers-events, which has no SEP + // number yet — see the header of src/seps/sep-9999.yaml. + new EventsDiscoveryScenario(), + new EventsPollScenario() ]; // All client scenarios @@ -223,6 +234,11 @@ const allClientScenariosList: ClientScenario[] = [ new SkillsEnumerationScenario(), new SkillsManifestScenario(), + // MCP Events. Fixture-dependent (needs a server declaring `capabilities.events`); + // each scenario SKIPs cleanly when the capability is not declared. + new EventsDiscoveryScenario(), + new EventsPollScenario(), + // Prompts scenarios new PromptsListScenario(), new PromptsGetSimpleScenario(), diff --git a/src/scenarios/server/events/discovery.ts b/src/scenarios/server/events/discovery.ts new file mode 100644 index 00000000..16cb0a15 --- /dev/null +++ b/src/scenarios/server/events/discovery.ts @@ -0,0 +1,548 @@ +/** + * MCP Events — capability declaration, `events/list`, and the error-code + * contract. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec excerpt lives next to its check ID in + * src/seps/sep-9999.yaml, keeping the yaml and this scenario in lock-step. + * 9999 is a placeholder SEP number; see that file's header. + * + * This is the gate scenario for the suite: the delivery-mode scenarios all + * start from a descriptor found here, so a server that fails `events/list` + * fails everything downstream for a reason this scenario names. + * + * Discovery is dynamic and brand-neutral. The scenario enumerates whatever the + * server serves and validates the descriptors it finds, hardcoding no event + * name. An empty catalog is permitted by the document, so descriptor-level + * checks report the unmet prerequisite via `untestableCheck` rather than + * passing vacuously. + * + * The capability gate is two-sided rather than a plain SKIP. An optional + * capability a server never declared is not a defect, so a server that also + * does not implement `events/list` skips the whole scenario. A server that + * answers `events/list` while declaring nothing is a different thing: it has + * an events surface that a client following the spec would never call, and + * SKIP would report that as a clean run. So the scenario asks before it + * skips. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_EXTENSION_ID, + EVENTS_CAPABILITY, + EVENTS_LIST_METHOD, + EVENTS_POLL_METHOD, + EVENTS_SPEC_REF, + EVENTS_NOT_FOUND, + EVENTS_UNSUPPORTED, + DELIVERY_MODES, + JSONRPC_METHOD_NOT_FOUND, + type EventDescriptor, + declaredEventsCapability, + eventsCapability, + eventsCheck, + eventsListPage, + eventsListAll, + eventsPoll, + deliveryModes, + descriptorName, + descriptorLabel, + describeValue, + isObject, + inServerErrorRange, + unknownEventName +} from './helpers'; + +const CAPABILITY_IDS = [ + 'sep-9999-capability-events-object', + 'sep-9999-capability-list-changed-flag' +] as const; + +const LIST_IDS = [ + 'sep-9999-list-implemented', + 'sep-9999-list-pagination' +] as const; + +const DESCRIPTOR_IDS = [ + 'sep-9999-descriptor-name', + 'sep-9999-descriptor-description', + 'sep-9999-descriptor-delivery-subset', + 'sep-9999-descriptor-input-schema', + 'sep-9999-descriptor-payload-schema', + 'sep-9999-descriptor-meta' +] as const; + +const ERROR_IDS = [ + 'sep-9999-error-not-found', + 'sep-9999-error-server-range' +] as const; + +const ALL_IDS = [ + ...CAPABILITY_IDS, + ...LIST_IDS, + ...DESCRIPTOR_IDS, + ...ERROR_IDS +]; + +/** Every check this scenario can emit, as SKIPPED with one shared reason. */ +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsDiscoveryScenario implements ClientScenario { + name = 'events-discovery'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: capability declaration, \`events/list\` enumeration, and the error-code contract. + +**Methods**: \`events/list\` (mandatory for a server declaring \`capabilities.events\`), \`events/poll\` (probed only for its error path) + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-capability-events-object\` — \`events\` is declared top-level under \`capabilities\` as an object +- \`sep-9999-capability-list-changed-flag\` — \`listChanged\`, when present, is a boolean +- \`sep-9999-list-implemented\` — \`events/list\` is implemented and returns an \`events\` array +- \`sep-9999-list-pagination\` — \`nextCursor\` is honoured as a cursor on the next request +- \`sep-9999-descriptor-*\` — the descriptor fields: \`name\`, \`description\`, \`delivery\` as a non-empty subset of poll/push/webhook, \`inputSchema\`, \`payloadSchema\`, and \`_meta\` when present +- \`sep-9999-error-not-found\` — an unknown event name answers \`-32011 NotFound\` (the poll-specific restatement of the same rule is graded by \`events-poll\`) +- \`sep-9999-error-server-range\` — the extension's codes sit in the JSON-RPC implementation-defined server range + +**Discovery is dynamic**: a server that neither declares the capability nor implements \`events/list\` SKIPs everything. One that answers \`events/list\` without declaring the capability is graded, and fails the declaration check, because that surface is unreachable for a client that reads capabilities first. An empty catalog reports the descriptor checks as untestable rather than passing them.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + return await this.checks(conn); + } finally { + await conn.close(); + } + } + + private async checks( + conn: Awaited> + ): Promise { + const checks: ConformanceCheck[] = []; + + // --- Capability ------------------------------------------------------ + const capDescription = + 'Servers advertise event support in their capabilities as an object under `capabilities.events`.'; + const { declared, value } = await declaredEventsCapability(conn); + + if (!declared) { + // An undeclared optional capability is normally a SKIP. It is not one + // when the server answers `events/list` anyway: that server has an + // events surface no spec-following client can discover, and SKIP would + // report it as a clean run. Distinguish the two by asking. + const probe = await eventsListPage(conn); + if ('error' in probe && probe.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + checks.push( + eventsCheck( + 'sep-9999-capability-events-object', + capDescription, + 'FAILURE', + { + errorMessage: `Server answers \`${EVENTS_LIST_METHOD}\` but declares no \`capabilities.${EVENTS_CAPABILITY}\`. A client that follows the spec reads capabilities to decide whether to call it, so this surface is unreachable.`, + details: { capabilities: EVENTS_CAPABILITY, declared: false } + } + ) + ); + } else if (isObject(value)) { + checks.push( + eventsCheck( + 'sep-9999-capability-events-object', + capDescription, + 'SUCCESS' + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-9999-capability-events-object', + capDescription, + 'FAILURE', + { + errorMessage: `\`capabilities.${EVENTS_CAPABILITY}\` is ${describeValue(value)}, expected an object.`, + details: { declared: value } + } + ) + ); + } + + const caps = declared ? await eventsCapability(conn) : undefined; + const listChanged = caps?.listChanged; + if (listChanged === undefined) { + checks.push( + eventsCheck( + 'sep-9999-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'SKIPPED', + { + errorMessage: declared + ? 'Server did not declare `listChanged`; the flag is optional and its absence means the notification is not advertised.' + : 'Server declared no `capabilities.events` object for the flag to sit in; see sep-9999-capability-events-object.' + } + ) + ); + } else if (typeof listChanged === 'boolean') { + checks.push( + eventsCheck( + 'sep-9999-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'SUCCESS', + { details: { listChanged } } + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-9999-capability-list-changed-flag', + 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', + 'FAILURE', + { + errorMessage: `\`capabilities.events.listChanged\` is ${describeValue(listChanged)}, expected a boolean.`, + details: { listChanged } + } + ) + ); + } + + // --- events/list ----------------------------------------------------- + const firstPage = await eventsListPage(conn); + if ('error' in firstPage) { + const err = firstPage.error; + const unimplemented = err.code === JSONRPC_METHOD_NOT_FOUND; + checks.push( + eventsCheck( + 'sep-9999-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'FAILURE', + { + errorMessage: unimplemented + ? `Server declares \`capabilities.events\` but \`${EVENTS_LIST_METHOD}\` is not implemented (-32601).` + : `\`${EVENTS_LIST_METHOD}\` failed: ${err.code} ${err.message}`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + // Nothing downstream can be graded without a catalog. + const reason = `\`${EVENTS_LIST_METHOD}\` did not return a result (${err.code} ${err.message}).`; + for (const id of [...LIST_IDS.slice(1), ...DESCRIPTOR_IDS]) { + checks.push( + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], 'FAILURE') + ); + } + checks.push(...(await this.errorChecks(conn, []))); + return checks; + } + + const eventsField = firstPage.result.events; + if (Array.isArray(eventsField)) { + checks.push( + eventsCheck( + 'sep-9999-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'SUCCESS', + { details: { firstPageCount: firstPage.descriptors.length } } + ) + ); + } else { + checks.push( + eventsCheck( + 'sep-9999-list-implemented', + '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.', + 'FAILURE', + { + errorMessage: `\`${EVENTS_LIST_METHOD}\` result \`events\` is ${describeValue(eventsField)}, expected an array.`, + details: { events: eventsField } + } + ) + ); + } + + checks.push(await this.paginationCheck(conn, firstPage.result.nextCursor)); + + const all = await eventsListAll(conn); + const descriptors = + 'error' in all ? firstPage.descriptors : all.descriptors; + checks.push(...this.descriptorChecks(descriptors)); + checks.push(...(await this.errorChecks(conn, descriptors))); + + return checks; + } + + /** + * `nextCursor` is only gradeable when the server actually paginates. A + * single-page catalog leaves nothing to follow, which is a missing + * prerequisite rather than a pass: the document defers the semantics to the + * base protocol, so the only thing this check can establish is that + * `events/list` participates in the scheme at all. + */ + private async paginationCheck( + conn: Awaited>, + nextCursor: unknown + ): Promise { + const id = 'sep-9999-list-pagination'; + const description = + '`nextCursor` is present when more pages are available; same semantics as tools/list.'; + + if (nextCursor === undefined || nextCursor === null) { + return untestableCheck( + id, + id, + description, + `Server returned a single page from \`${EVENTS_LIST_METHOD}\` with no \`nextCursor\`, so cursor round-tripping could not be exercised.`, + [EVENTS_SPEC_REF], + 'WARNING' + ); + } + + if (typeof nextCursor !== 'string' || nextCursor.length === 0) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `\`nextCursor\` is ${describeValue(nextCursor)}, expected a non-empty string.`, + details: { nextCursor } + }); + } + + const second = await eventsListPage(conn, nextCursor); + if ('error' in second) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Server returned \`nextCursor\` but rejected it on the next \`${EVENTS_LIST_METHOD}\`: ${second.error.code} ${second.error.message}`, + details: { nextCursor, code: second.error.code } + }); + } + + if (second.result.nextCursor === nextCursor) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: + 'Server echoed the same `nextCursor` on the following page, which never terminates.', + details: { nextCursor } + }); + } + + return eventsCheck(id, description, 'SUCCESS', { + details: { secondPageCount: second.descriptors.length } + }); + } + + /** + * Grade every descriptor and report one check per field, naming the first + * offender. One check per field rather than per descriptor keeps the check + * IDs stable across servers with different catalog sizes. + */ + private descriptorChecks(descriptors: EventDescriptor[]): ConformanceCheck[] { + if (descriptors.length === 0) { + const reason = `Server's \`${EVENTS_LIST_METHOD}\` returned an empty catalog, so no descriptor could be validated.`; + return DESCRIPTOR_IDS.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], 'FAILURE') + ); + } + + const out: ConformanceCheck[] = []; + + const field = ( + id: string, + description: string, + severity: 'FAILURE' | 'WARNING', + predicate: (d: EventDescriptor) => string | undefined + ) => { + for (const [i, d] of descriptors.entries()) { + const problem = predicate(d); + if (problem) { + out.push( + eventsCheck(id, description, severity, { + errorMessage: `${descriptorLabel(d, i)}: ${problem}`, + details: { descriptor: d } + }) + ); + return; + } + } + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { descriptorsChecked: descriptors.length } + }) + ); + }; + + field( + 'sep-9999-descriptor-name', + 'Each event descriptor carries a `name` identifying the event type.', + 'FAILURE', + (d) => + descriptorName(d) === undefined + ? `\`name\` is ${describeValue(d.name)}, expected a non-empty string.` + : undefined + ); + + field( + 'sep-9999-descriptor-description', + 'Each event descriptor carries a `description` of when the event fires.', + 'WARNING', + (d) => + typeof d.description === 'string' && d.description.length > 0 + ? undefined + : `\`description\` is ${describeValue(d.description)}, expected a non-empty string.` + ); + + field( + 'sep-9999-descriptor-delivery-subset', + '`delivery` lists the delivery modes this event type supports — any non-empty subset of `poll`, `push`, `webhook`.', + 'FAILURE', + (d) => { + if (!Array.isArray(d.delivery)) { + return `\`delivery\` is ${describeValue(d.delivery)}, expected an array.`; + } + const modes = deliveryModes(d); + if (modes.length === 0) + return '`delivery` is empty; the subset must be non-empty.'; + const unknown = modes.filter( + (m) => !(DELIVERY_MODES as readonly string[]).includes(m) + ); + if (unknown.length > 0) { + return `\`delivery\` contains ${unknown.map((m) => `\`${m}\``).join(', ')}, outside the poll/push/webhook set.`; + } + if (new Set(modes).size !== modes.length) { + return '`delivery` repeats a mode; it is a subset, not a list.'; + } + return undefined; + } + ); + + field( + 'sep-9999-descriptor-input-schema', + '`inputSchema` is a JSON Schema describing valid subscription arguments.', + 'FAILURE', + (d) => + isObject(d.inputSchema) + ? undefined + : `\`inputSchema\` is ${describeValue(d.inputSchema)}, expected a JSON Schema object.` + ); + + field( + 'sep-9999-descriptor-payload-schema', + '`payloadSchema` describes the shape of `data` in delivered events.', + 'FAILURE', + (d) => + isObject(d.payloadSchema) + ? undefined + : `\`payloadSchema\` is ${describeValue(d.payloadSchema)}, expected a JSON Schema object.` + ); + + field( + 'sep-9999-descriptor-meta', + '`_meta` on an event descriptor is optional; same semantics as on Tool/Resource/Prompt.', + 'WARNING', + (d) => + d._meta === undefined || isObject(d._meta) + ? undefined + : `\`_meta\` is ${describeValue(d._meta)}, expected an object when present.` + ); + + return out; + } + + /** + * Probe the unknown-name error path through `events/poll`. + * + * Poll is the cheapest probe: it holds no server-side state, so a rejected + * call leaves nothing behind. The document states the same obligation twice, + * once as the general `-32011 NotFound` code and once as the poll-specific + * consequence of an event type having been removed, so both rows are graded + * from this one exchange rather than by poking the server twice. + */ + private async errorChecks( + conn: Awaited>, + descriptors: EventDescriptor[] + ): Promise { + const notFoundDesc = + '`-32011 NotFound` — a referenced entity does not exist, such as an unknown event name.'; + const rangeDesc = + "The extension's codes are carried in the JSON-RPC implementation-defined server range `[-32000, -32099]`."; + + // A server offering no poll-capable event type may legitimately not route + // events/poll at all, which would make -32601 the honest answer and this + // probe meaningless. + const pollable = descriptors.some((d) => deliveryModes(d).includes('poll')); + if (descriptors.length > 0 && !pollable) { + const reason = `No event type advertises \`poll\` delivery, so \`${EVENTS_POLL_METHOD}\` could not be used to probe the unknown-name error path.`; + return [ + untestableCheck( + 'sep-9999-error-not-found', + 'sep-9999-error-not-found', + notFoundDesc, + reason, + [EVENTS_SPEC_REF], + 'FAILURE' + ), + untestableCheck( + 'sep-9999-error-server-range', + 'sep-9999-error-server-range', + rangeDesc, + reason, + [EVENTS_SPEC_REF], + 'FAILURE' + ) + ]; + } + + const name = unknownEventName(); + const probe = await eventsPoll(conn, { name, arguments: {}, cursor: null }); + + if (!('error' in probe)) { + const errorMessage = `\`${EVENTS_POLL_METHOD}\` for unknown event name \`${name}\` returned a result instead of an error.`; + return [ + eventsCheck('sep-9999-error-not-found', notFoundDesc, 'FAILURE', { + errorMessage, + details: { result: probe.result } + }), + eventsCheck('sep-9999-error-server-range', rangeDesc, 'FAILURE', { + errorMessage + }) + ]; + } + + const { code, message, data } = probe.error; + const isNotFound = code === EVENTS_NOT_FOUND; + const details = { code, message, data, probedName: name }; + + const out: ConformanceCheck[] = []; + + out.push( + isNotFound + ? eventsCheck('sep-9999-error-not-found', notFoundDesc, 'SUCCESS', { + details + }) + : eventsCheck('sep-9999-error-not-found', notFoundDesc, 'FAILURE', { + errorMessage: `Unknown event name answered ${code}, expected ${EVENTS_NOT_FOUND} NotFound.${ + code === EVENTS_UNSUPPORTED + ? ' `-32014 Unsupported` is for a well-formed request naming an option the server does not offer, not for a name it does not have.' + : '' + }`, + details + }) + ); + + out.push( + inServerErrorRange(code) + ? eventsCheck('sep-9999-error-server-range', rangeDesc, 'SUCCESS', { + details + }) + : eventsCheck('sep-9999-error-server-range', rangeDesc, 'FAILURE', { + errorMessage: `Error code ${code} is outside the implementation-defined server range [-32099, -32000].`, + details + }) + ); + + return out; + } +} + +/** Exported for the negative tests, which assert the full emitted set. */ +export const EVENTS_DISCOVERY_CHECK_IDS = ALL_IDS; diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts new file mode 100644 index 00000000..22cfa7c6 --- /dev/null +++ b/src/scenarios/server/events/helpers.ts @@ -0,0 +1,391 @@ +/** + * Shared helpers for the MCP Events server-conformance scenarios under this + * directory. + * + * Extracted against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events (merged 2026-09-08). + * Each check's verbatim excerpt lives next to its check ID in + * src/seps/sep-9999.yaml, and 9999 is a placeholder SEP number — see that + * file's header before renaming anything here. + * + * Two things about Events differ from every other extension suite here and are + * worth knowing before reading the scenarios: + * + * 1. The capability is declared at the top level as `capabilities.events`, not + * under `capabilities.extensions`. SEP-2133's extensions map does not come + * into it. `EVENTS_EXTENSION_ID` exists only as a `ScenarioSource` key so + * the runner keeps these scenarios off the `--spec-version` timeline; it is + * never a path into the capability object. + * + * 2. No delivery mode is mandatory. A descriptor's `delivery` array is any + * non-empty subset of poll/push/webhook, so a scenario for one mode has to + * discover whether any event type offers it before it can probe anything. + * Nothing is hardcoded to a fixture's event names. + */ + +import type { + CheckStatus, + ConformanceCheck, + SpecReference +} from '../../../types'; +import type { Connection } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; + +/** + * Suite-selection key for the Events scenarios. + * + * Events has no SEP-2133 extension identifier, because it declares its + * capability top-level rather than inside `capabilities.extensions`. This + * string exists so `ScenarioSource` can carry `{ extensionId }`, which is what + * keeps the scenarios out of `--spec-version` selection (see + * `matchesSpecVersion` in src/scenarios/index.ts). Do not read the capability + * at this key; read `capabilities.events`. + */ +export const EVENTS_EXTENSION_ID = 'io.modelcontextprotocol/events'; + +/** The capability key, top-level under `capabilities`. */ +export const EVENTS_CAPABILITY = 'events'; + +export const EVENTS_LIST_METHOD = 'events/list'; +export const EVENTS_POLL_METHOD = 'events/poll'; +export const EVENTS_STREAM_METHOD = 'events/stream'; +export const EVENTS_SUBSCRIBE_METHOD = 'events/subscribe'; +export const EVENTS_UNSUBSCRIBE_METHOD = 'events/unsubscribe'; + +export const EVENTS_LIST_CHANGED_NOTIFICATION = + 'notifications/events/list_changed'; +export const EVENTS_EVENT_NOTIFICATION = 'notifications/events/event'; +export const EVENTS_ACTIVE_NOTIFICATION = 'notifications/events/active'; +export const EVENTS_HEARTBEAT_NOTIFICATION = 'notifications/events/heartbeat'; +export const EVENTS_ERROR_NOTIFICATION = 'notifications/events/error'; +export const EVENTS_TERMINATED_NOTIFICATION = 'notifications/events/terminated'; + +/** + * The `_meta` key carrying the parent `events/stream` request id on every + * `notifications/events/*` message, per SEP-2575's correlation convention. + */ +export const SUBSCRIPTION_ID_META = 'io.modelcontextprotocol/subscriptionId'; + +/** The three delivery modes, as they appear in a descriptor's `delivery`. */ +export const DELIVERY_MODES = ['poll', 'push', 'webhook'] as const; +export type DeliveryMode = (typeof DELIVERY_MODES)[number]; + +/** Standard JSON-RPC. */ +export const JSONRPC_METHOD_NOT_FOUND = -32601; +export const JSONRPC_INVALID_PARAMS = -32602; + +/** + * The general-purpose codes this document defines, carried in the JSON-RPC + * implementation-defined server range. Named for reuse across MCP rather than + * scoped to events, and each conveys its specifics through a typed `data` + * payload rather than by minting more numbers. + */ +export const EVENTS_NOT_FOUND = -32011; +export const EVENTS_FORBIDDEN = -32012; +export const EVENTS_RESOURCE_EXHAUSTED = -32013; +export const EVENTS_UNSUPPORTED = -32014; +export const EVENTS_CALLBACK_ENDPOINT_ERROR = -32015; + +/** Inclusive bounds of the JSON-RPC implementation-defined server range. */ +export const SERVER_ERROR_RANGE_MIN = -32099; +export const SERVER_ERROR_RANGE_MAX = -32000; + +export const EVENTS_SPEC_REF: SpecReference = { + id: 'MCP-Events', + url: 'https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md' +}; + +/** One entry of the `events` array returned by `events/list`. */ +export interface EventDescriptor { + name?: unknown; + description?: unknown; + delivery?: unknown; + inputSchema?: unknown; + payloadSchema?: unknown; + _meta?: unknown; + [key: string]: unknown; +} + +export interface EventsListResult { + events?: unknown; + nextCursor?: unknown; + [key: string]: unknown; +} + +/** One entry of a poll response's `events` array, or a pushed notification. */ +export interface EventOccurrence { + eventId?: unknown; + name?: unknown; + timestamp?: unknown; + data?: unknown; + cursor?: unknown; + _meta?: unknown; + [key: string]: unknown; +} + +export interface EventsPollResult { + events?: unknown; + cursor?: unknown; + truncated?: unknown; + hasMore?: unknown; + nextPollMs?: unknown; + [key: string]: unknown; +} + +/** + * Build a check carrying the Events spec reference. Per AGENTS.md the same + * `id` flips `status` + `errorMessage` between SUCCESS and FAILURE rather than + * branching into distinct slugs. + */ +export function eventsCheck( + id: string, + description: string, + status: CheckStatus, + extras: Partial = {} +): ConformanceCheck { + return { + id, + name: id, + description, + status, + timestamp: new Date().toISOString(), + specReferences: [EVENTS_SPEC_REF], + ...extras + }; +} + +/** A JSON object, as opposed to an array, `null`, or a primitive. */ +export function isObject(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** + * How to name an observed value in an error message. + * + * `absent` rather than `a undefined`, because a reader chasing a failure needs + * to know the field was missing, and the JavaScript spelling of that is noise. + */ +export function describeValue(value: unknown): string { + if (value === undefined) return 'absent'; + if (value === null) return 'null'; + if (Array.isArray(value)) return 'an array'; + return `a ${typeof value}`; +} + +/** + * Whether the server declared the events capability at all, and the raw value + * it declared it with, before any shape coercion. + * + * Kept separate from `eventsCapability` so callers can tell "absent" from + * "declared with the wrong type" apart. Folding the two together would turn a + * server that declares `events: true` into a clean SKIP of the whole suite, + * which reads as a green run against a server that is plainly wrong. + */ +export async function declaredEventsCapability( + conn: Connection +): Promise<{ declared: boolean; value: unknown }> { + const discovered = await conn.discover(); + const caps = (discovered.capabilities as Record) ?? {}; + if (!(EVENTS_CAPABILITY in caps)) + return { declared: false, value: undefined }; + return { declared: true, value: caps[EVENTS_CAPABILITY] }; +} + +/** + * The events capability object, or `undefined` when the server did not declare + * it — or declared it with something that is not an object, which callers + * treat the same way. An undeclared optional capability is a SKIP. + */ +export async function eventsCapability( + conn: Connection +): Promise | undefined> { + const { value } = await declaredEventsCapability(conn); + return isObject(value) ? value : undefined; +} + +/** A single `events/list` page, kept separate so pagination can be inspected. */ +export interface EventsListPage { + result: EventsListResult; + descriptors: EventDescriptor[]; +} + +/** + * Call `events/list` once, optionally with a cursor. Returns the `JsonRpcError` + * rather than throwing, so a scenario can grade the error instead of aborting. + */ +export async function eventsListPage( + conn: Connection, + cursor?: string +): Promise { + try { + const result = await conn.request( + EVENTS_LIST_METHOD, + cursor ? { cursor } : undefined + ); + const raw = result?.events; + return { + result: result ?? {}, + descriptors: Array.isArray(raw) ? (raw as EventDescriptor[]) : [] + }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } +} + +/** + * Every descriptor from `events/list`, paginating until `nextCursor` clears. + * + * Bounded at `maxPages` because a server that echoes the same `nextCursor` + * forever would otherwise hang the scenario rather than fail it. Hitting the + * bound is reported through `truncatedByBound` so the caller can say so instead + * of silently grading a partial catalog. + */ +export async function eventsListAll( + conn: Connection, + maxPages = 20 +): Promise< + | { descriptors: EventDescriptor[]; pages: number; truncatedByBound: boolean } + | { error: JsonRpcError } +> { + const out: EventDescriptor[] = []; + const seen = new Set(); + let cursor: string | undefined; + let pages = 0; + + do { + const page = await eventsListPage(conn, cursor); + if ('error' in page) return page; + pages += 1; + out.push(...page.descriptors); + + const next = page.result.nextCursor; + if (typeof next !== 'string' || next.length === 0) break; + // A repeated cursor is a server bug; stop rather than loop forever. The + // pagination check grades it, this helper just refuses to hang. + if (seen.has(next)) break; + seen.add(next); + cursor = next; + } while (pages < maxPages); + + return { descriptors: out, pages, truncatedByBound: pages >= maxPages }; +} + +/** The `delivery` array of a descriptor, or `[]` when it is missing/malformed. */ +export function deliveryModes(descriptor: EventDescriptor): string[] { + const d = descriptor.delivery; + return Array.isArray(d) + ? d.filter((m): m is string => typeof m === 'string') + : []; +} + +/** The first descriptor advertising `mode`, or `undefined` when none does. */ +export function firstSupporting( + descriptors: EventDescriptor[], + mode: DeliveryMode +): EventDescriptor | undefined { + return descriptors.find((d) => deliveryModes(d).includes(mode)); +} + +/** A descriptor's `name` when it is a usable string, else `undefined`. */ +export function descriptorName( + descriptor: EventDescriptor +): string | undefined { + return typeof descriptor.name === 'string' && descriptor.name.length > 0 + ? descriptor.name + : undefined; +} + +/** + * How to refer to a descriptor in an error message without assuming it has a + * usable `name` — the checks that grade `name` itself run against descriptors + * that may not. + */ +export function descriptorLabel( + descriptor: EventDescriptor, + index: number +): string { + const name = descriptorName(descriptor); + return name ? `\`${name}\`` : `events[${index}]`; +} + +/** + * Arguments that satisfy a descriptor's `inputSchema` well enough to poll with. + * + * Deliberately minimal: an empty object. Every `inputSchema` in the document is + * an object schema whose properties are filters and transforms, none of them + * required, so `{}` means "no filtering" and is valid against all of them. A + * schema that does declare `required` is the one case this cannot satisfy, and + * the caller reports that as an unmet prerequisite rather than guessing values + * a server would then reject for the wrong reason. + */ +export function minimalArguments( + descriptor: EventDescriptor +): Record | undefined { + const schema = descriptor.inputSchema; + if (!isObject(schema)) return {}; + const required = schema.required; + if (Array.isArray(required) && required.length > 0) return undefined; + return {}; +} + +/** + * Call `events/poll`, returning the `JsonRpcError` rather than throwing so the + * caller can grade error codes. + */ +export async function eventsPoll( + conn: Connection, + params: Record +): Promise<{ result: EventsPollResult } | { error: JsonRpcError }> { + try { + const result = await conn.request( + EVENTS_POLL_METHOD, + params + ); + return { result: result ?? {} }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } +} + +/** The `events` array of a poll result, or `[]` when missing/malformed. */ +export function occurrences(result: EventsPollResult): EventOccurrence[] { + return Array.isArray(result.events) + ? (result.events as EventOccurrence[]) + : []; +} + +/** + * Whether a value is an acceptable cursor: a string, `null`, or absent. + * + * "Absent means null" is normative in both directions, so a missing field is + * not a defect and callers must not treat it as one. + */ +export function isValidCursor(value: unknown): boolean { + return value === undefined || value === null || typeof value === 'string'; +} + +/** Whether a value parses as an ISO 8601 instant. */ +export function isIso8601(value: unknown): boolean { + if (typeof value !== 'string') return false; + const t = Date.parse(value); + if (Number.isNaN(t)) return false; + // Date.parse accepts bare dates and a few non-ISO forms; require at least a + // date and a time separated by `T`, which every example in the document has. + return /^\d{4}-\d{2}-\d{2}T/.test(value); +} + +/** Whether `code` sits in the JSON-RPC implementation-defined server range. */ +export function inServerErrorRange(code: number): boolean { + return code >= SERVER_ERROR_RANGE_MIN && code <= SERVER_ERROR_RANGE_MAX; +} + +/** + * A name no conformant server should be serving, for probing the "unknown + * event name" error path. Randomised so a fixture cannot accidentally define + * it, and prefixed so a human reading server logs knows where it came from. + */ +export function unknownEventName(): string { + return `conformance.nonexistent.${Math.random().toString(36).slice(2, 10)}`; +} diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts new file mode 100644 index 00000000..30d0622c --- /dev/null +++ b/src/scenarios/server/events/negative.test.ts @@ -0,0 +1,599 @@ +import { describe, test, expect } from 'vitest'; +import { createServer, type IncomingMessage, type Server } from 'http'; +import type { AddressInfo } from 'net'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { EventsDiscoveryScenario } from './discovery'; +import { EventsPollScenario } from './poll'; + +/** + * Negative controls for the MCP Events scenarios. + * + * A passing run against a conformant fixture proves a check does not + * false-positive. It does not prove the check catches anything, which is what + * these tests are for: every assertion below pairs a conformant server with a + * server broken in exactly one way, and asserts the check flips. + * + * The three divergences the suite is expected to find in mcpkit each get a + * test here, so the checks that will report them are known to work before + * anyone reads a red run and wonders whether the harness is wrong: + * `nextPollSeconds` (gap G31), and the two error-code paths behind event-type + * removal (gap G30). + * + * The fixture is a minimal SEP-2575 stateless server built per test rather + * than a checked-in example file, matching the SEP-2640 negative tests. An + * events-capable example server is a larger piece of work and belongs with the + * push and webhook scenarios, which genuinely need one. + */ + +/** A descriptor that is well formed apart from whatever a test overrides. */ +function descriptor(overrides: Record = {}) { + return { + name: 'test.event', + description: 'A negative-control fixture event type.', + delivery: ['poll'], + inputSchema: { + type: 'object', + properties: { channel: { type: 'string' } } + }, + payloadSchema: { type: 'object', properties: { id: { type: 'string' } } }, + ...overrides + }; +} + +/** A poll result that is well formed apart from whatever a test overrides. */ +function pollResult(overrides: Record = {}) { + return { + events: [], + cursor: 'cursor_001', + truncated: false, + hasMore: false, + nextPollMs: 30000, + ...overrides + }; +} + +/** An occurrence that is well formed apart from whatever a test overrides. */ +function occurrence(overrides: Record = {}) { + return { + eventId: 'evt_001', + name: 'test.event', + timestamp: '2026-09-15T12:00:00Z', + data: { id: 'x' }, + ...overrides + }; +} + +interface FixtureOptions { + /** Raw value to declare at `capabilities.events`; omit for no declaration. */ + capability?: unknown; + descriptors?: object[]; + /** Answer `events/list` with this JSON-RPC error instead of a result. */ + listError?: { code: number; message: string }; + /** + * Poll responses, consumed in order; the last one repeats once exhausted. + * A `{ error }` entry makes that poll answer with a JSON-RPC error. + */ + pollResponses?: Array< + Record | { error: { code: number; message: string } } + >; + /** Overrides keyed by the polled event name, taking priority over the queue. */ + pollByName?: Record< + string, + Record | { error: { code: number; message: string } } + >; + /** Error code for a poll naming an event type the fixture does not serve. */ + unknownNameCode?: number; + /** Error code for a poll whose arguments violate `inputSchema`. */ + invalidArgsCode?: number; +} + +function startFixture(opts: FixtureOptions): Promise<{ + url: string; + server: Server; + polls: Array>; +}> { + const polls: Array> = []; + const queue = [...(opts.pollResponses ?? [pollResult()])]; + const descriptors = opts.descriptors ?? [descriptor()]; + const names = new Set( + descriptors + .map((d) => (d as { name?: unknown }).name) + .filter((n): n is string => typeof n === 'string') + ); + + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const params = (body.params ?? {}) as Record; + + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (code: number, message: string, data?: unknown) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ jsonrpc: '2.0', id, error: { code, message, data } }) + ); + }; + + if (method === 'server/discover') { + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: 'capability' in opts ? { events: opts.capability } : {}, + serverInfo: { name: 'events-negative', version: '1.0.0' } + }); + return; + } + + if (method === 'events/list') { + if (opts.listError) { + fail(opts.listError.code, opts.listError.message); + return; + } + send({ events: descriptors }); + return; + } + + if (method === 'events/poll') { + polls.push(params); + const name = params.name; + + if (typeof name !== 'string') { + fail(-32602, 'InvalidParams: `name` is required'); + return; + } + if (!names.has(name)) { + fail(opts.unknownNameCode ?? -32011, 'NotFound', { kind: 'event' }); + return; + } + + const byName = opts.pollByName?.[name]; + const chosen = + byName ?? + (queue.length > 1 ? queue.shift()! : (queue[0] ?? pollResult())); + + // Argument validation against the fixture's own declared schema, so the + // invalid-arguments probe has something real to violate. + const args = (params.arguments ?? {}) as Record; + const decl = descriptors.find( + (d) => (d as { name?: unknown }).name === name + ) as { inputSchema?: { properties?: Record } }; + for (const [key, value] of Object.entries(args)) { + const declared = decl?.inputSchema?.properties?.[key]; + if (declared?.type === 'string' && typeof value !== 'string') { + fail(opts.invalidArgsCode ?? -32602, 'InvalidParams'); + return; + } + } + + if ('error' in chosen) { + const e = (chosen as { error: { code: number; message: string } }) + .error; + fail(e.code, e.message); + return; + } + send(chosen as Record); + return; + } + + fail(-32601, `Method not found: ${method}`); + }); + + return new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, () => { + const addr = server.address() as AddressInfo; + resolve({ url: `http://localhost:${addr.port}/mcp`, server, polls }); + }); + }); +} + +async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} + +type Scenario = EventsDiscoveryScenario | EventsPollScenario; + +async function checksFor(scenario: Scenario, opts: FixtureOptions) { + const { url, server } = await startFixture(opts); + try { + const checks = await scenario.run(testContext(url, DRAFT_PROTOCOL_VERSION)); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await new Promise((r) => server.close(() => r())); + } +} + +const discovery = () => new EventsDiscoveryScenario(); +const poll = () => new EventsPollScenario(); + +/** The baseline every negative case is compared against. */ +const CONFORMANT: FixtureOptions = { + capability: { listChanged: true }, + descriptors: [descriptor()], + pollResponses: [pollResult()] +}; + +describe('events capability declaration', () => { + test('an object declaration passes; a boolean one fails rather than skipping', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-9999-capability-events-object')?.status).toBe('SUCCESS'); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + capability: true + }); + const check = broken.get('sep-9999-capability-events-object'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('a boolean'); + }); + + test('a server that declares nothing and serves nothing skips the suite', async () => { + const checks = await checksFor(discovery(), { + listError: { code: -32601, message: 'Method not found' } + }); + for (const check of checks.values()) { + expect(check.status).toBe('SKIPPED'); + } + }); + + // The case mcpkit is actually in: events/list answers, but nothing is + // declared, so a client that reads capabilities first never calls it. A + // plain SKIP here would report that as a clean run. + test('serving events/list while declaring nothing fails rather than skipping', async () => { + const checks = await checksFor(discovery(), { + descriptors: [descriptor()] + }); + const cap = checks.get('sep-9999-capability-events-object'); + expect(cap?.status).toBe('FAILURE'); + expect(cap?.errorMessage).toContain('declares no `capabilities.events`'); + expect(cap?.errorMessage).toContain('unreachable'); + + // And the rest of the scenario is still graded, not abandoned. + expect(checks.get('sep-9999-list-implemented')?.status).toBe('SUCCESS'); + expect(checks.get('sep-9999-descriptor-name')?.status).toBe('SUCCESS'); + }); + + test('the poll scenario likewise grades an undeclared-but-serving server', async () => { + const checks = await checksFor(poll(), { descriptors: [descriptor()] }); + expect(checks.get('sep-9999-poll-implemented')?.status).toBe('SUCCESS'); + + const skipped = await checksFor(poll(), { + listError: { code: -32601, message: 'Method not found' } + }); + expect(skipped.get('sep-9999-poll-implemented')?.status).toBe('SKIPPED'); + }); + + test('a non-boolean listChanged fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + capability: { listChanged: 'yes' } + }); + const check = checks.get('sep-9999-capability-list-changed-flag'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('a string'); + }); +}); + +describe('events/list descriptors', () => { + test('a descriptor missing name fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ name: undefined })] + }); + expect(checks.get('sep-9999-descriptor-name')?.status).toBe('FAILURE'); + }); + + test('an empty delivery array fails; a valid subset passes', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-9999-descriptor-delivery-subset')?.status).toBe( + 'SUCCESS' + ); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ delivery: [] })] + }); + const check = broken.get('sep-9999-descriptor-delivery-subset'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('non-empty'); + }); + + test('a delivery mode outside poll/push/webhook fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ delivery: ['poll', 'carrier-pigeon'] })] + }); + const check = checks.get('sep-9999-descriptor-delivery-subset'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('carrier-pigeon'); + }); + + test('a missing payloadSchema fails', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [descriptor({ payloadSchema: undefined })] + }); + expect(checks.get('sep-9999-descriptor-payload-schema')?.status).toBe( + 'FAILURE' + ); + }); + + test('an empty catalog reports descriptor checks untestable, never green', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + descriptors: [] + }); + const check = checks.get('sep-9999-descriptor-name'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + expect(check?.details?.untestable).toBe(true); + }); + + test('an unimplemented events/list fails and names the capability mismatch', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + listError: { code: -32601, message: 'Method not found' } + }); + const check = checks.get('sep-9999-list-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('declares `capabilities.events`'); + }); +}); + +describe('events error codes', () => { + test('an unknown event name answering -32011 passes, -32014 fails', async () => { + const ok = await checksFor(discovery(), CONFORMANT); + expect(ok.get('sep-9999-error-not-found')?.status).toBe('SUCCESS'); + + const broken = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -32014 + }); + const check = broken.get('sep-9999-error-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32014 Unsupported` is for'); + }); + + test('a code outside the server range fails the range check', async () => { + const checks = await checksFor(discovery(), { + ...CONFORMANT, + unknownNameCode: -1 + }); + const check = checks.get('sep-9999-error-server-range'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('outside'); + }); +}); + +describe('events/poll response shape', () => { + test('nextPollSeconds is caught and named as the pre-rename field', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-poll-next-poll-ms')?.status).toBe('SUCCESS'); + + // Gap G31: mcpkit still ships this. The check has to name the rename, or a + // reader of the red run cannot tell it from a server that omits the field. + const legacy = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ nextPollMs: undefined, nextPollSeconds: 30 }) + ] + }); + const check = legacy.get('sep-9999-poll-next-poll-ms'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('nextPollSeconds'); + expect(check?.errorMessage).toContain('197c32b4'); + }); + + test('a non-array events field fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ events: null })] + }); + expect(checks.get('sep-9999-poll-events-array')?.status).toBe('FAILURE'); + }); + + test('a numeric cursor fails the opaque-string check', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ cursor: 42 })] + }); + expect(checks.get('sep-9999-poll-response-cursor')?.status).toBe('FAILURE'); + expect(checks.get('sep-9999-cursor-opaque')?.status).toBe('FAILURE'); + }); + + test('replaying history for a null cursor fails start-from-now', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult({ events: [occurrence()] })] + }); + const check = checks.get('sep-9999-cursor-null-starts-from-now'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('start from now'); + }); +}); + +describe('events/poll cursor lifecycle', () => { + test('a quiet poll that drops the cursor fails advancement', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-cursor-advances-when-quiet')?.status).toBe( + 'SUCCESS' + ); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult(), pollResult({ cursor: 7 })] + }); + expect(broken.get('sep-9999-cursor-advances-when-quiet')?.status).toBe( + 'FAILURE' + ); + }); + + test('a type that flips between a cursor and null warns on consistency', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [pollResult(), pollResult({ cursor: null })] + }); + const check = checks.get('sep-9999-cursor-consistency'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('branch once'); + }); + + test('rejecting a poll that omits cursor fails absent-means-null', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + // Poll order: 1 bootstraps, 2 carries the cursor forward, 3 is the one + // that omits `cursor` entirely. Only the third may fail here. + pollResponses: [ + pollResult(), + pollResult(), + { error: { code: -32602, message: 'InvalidParams: cursor required' } } + ] + }); + const check = checks.get('sep-9999-cursor-absent-equals-null'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('absent cursor means'); + }); + + test('truncated true with no fresh cursor fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult(), + pollResult(), + pollResult(), + pollResult({ truncated: true, cursor: null }) + ] + }); + const check = checks.get('sep-9999-truncated-returns-fresh-cursor'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('no fresh position'); + }); +}); + +describe('events/poll error contract', () => { + test('a poll omitting name must not return a result', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-poll-one-subscription-per-request')?.status).toBe( + 'SUCCESS' + ); + }); + + test('wrong-typed arguments answering -32011 fails the invalid-params check', async () => { + const ok = await checksFor(poll(), CONFORMANT); + expect(ok.get('sep-9999-poll-invalid-arguments')?.status).toBe('SUCCESS'); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + invalidArgsCode: -32011 + }); + const check = broken.get('sep-9999-poll-invalid-arguments'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected -32602'); + }); + + test('polling a push-only type must answer -32014, not a result', async () => { + const pushOnly = descriptor({ name: 'push.only', delivery: ['push'] }); + const broken = await checksFor(poll(), { + ...CONFORMANT, + descriptors: [descriptor(), pushOnly], + pollByName: { 'push.only': pollResult() } + }); + const check = broken.get('sep-9999-poll-mode-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('does not advertise'); + + const ok = await checksFor(poll(), { + ...CONFORMANT, + descriptors: [descriptor(), pushOnly], + pollByName: { + 'push.only': { error: { code: -32014, message: 'Unsupported' } } + } + }); + expect(ok.get('sep-9999-poll-mode-unsupported')?.status).toBe('SUCCESS'); + }); + + test('every type offering poll reports the unsupported-mode check untestable', async () => { + const checks = await checksFor(poll(), CONFORMANT); + const check = checks.get('sep-9999-poll-mode-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + }); +}); + +describe('EventOccurrence shape', () => { + test('a quiet server reports the occurrence checks untestable, never green', async () => { + const checks = await checksFor(poll(), CONFORMANT); + const check = checks.get('sep-9999-occurrence-event-id'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('Not testable'); + }); + + test('a non-ISO timestamp fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence({ timestamp: 'last tuesday' })], + cursor: null + }) + ] + }); + expect(checks.get('sep-9999-occurrence-timestamp')?.status).toBe('FAILURE'); + }); + + test('a repeated eventId within one batch warns', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence(), occurrence()], + cursor: null + }) + ] + }); + const check = checks.get('sep-9999-occurrence-event-id-from-upstream'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('dedup'); + }); + + test('a missing data object fails', async () => { + const checks = await checksFor(poll(), { + ...CONFORMANT, + pollResponses: [ + pollResult({ + events: [occurrence({ data: undefined })], + cursor: null + }) + ] + }); + expect(checks.get('sep-9999-occurrence-data')?.status).toBe('FAILURE'); + }); +}); diff --git a/src/scenarios/server/events/poll.ts b/src/scenarios/server/events/poll.ts new file mode 100644 index 00000000..74a8845a --- /dev/null +++ b/src/scenarios/server/events/poll.ts @@ -0,0 +1,1208 @@ +/** + * MCP Events — poll delivery, the `EventOccurrence` shape, and cursor + * lifecycle. + * + * One scenario, many checks (per AGENTS.md "fewer scenarios, more checks"). + * Each check's verbatim spec excerpt lives next to its check ID in + * src/seps/sep-9999.yaml. 9999 is a placeholder SEP number; see that file's + * header. + * + * Poll is the mode a conformance harness can exercise completely. It is + * request/response, it holds no server-side state, and every poll response + * carries the cursor, so cursor advancement and `truncated` are observable + * without waiting for anything to happen upstream. Push needs a live stream + * and webhook needs a reachable callback; both are separate scenarios. + * + * The scenario drives a real quiet-period poll loop: two polls with the cursor + * from the first fed into the second. That is what makes + * `sep-9999-cursor-advances-when-quiet` gradeable against a server with no + * traffic, which is the state a conformance fixture is usually in. + * + * Nothing is hardcoded to a fixture's event names. The scenario picks the + * first descriptor advertising `poll` and works from there; when none does, + * every check reports the unmet prerequisite rather than passing vacuously. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_EXTENSION_ID, + EVENTS_POLL_METHOD, + EVENTS_SPEC_REF, + EVENTS_NOT_FOUND, + EVENTS_UNSUPPORTED, + JSONRPC_INVALID_PARAMS, + JSONRPC_METHOD_NOT_FOUND, + type EventDescriptor, + type EventOccurrence, + type EventsPollResult, + declaredEventsCapability, + eventsCheck, + eventsListAll, + eventsPoll, + occurrences, + deliveryModes, + descriptorName, + describeValue, + isObject, + isValidCursor, + isIso8601, + minimalArguments, + firstSupporting, + unknownEventName +} from './helpers'; + +const POLL_IDS = [ + 'sep-9999-poll-implemented', + 'sep-9999-poll-one-subscription-per-request', + 'sep-9999-poll-bootstraps-subscription', + 'sep-9999-poll-events-array', + 'sep-9999-poll-response-cursor', + 'sep-9999-poll-next-poll-ms', + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + 'sep-9999-poll-has-more', + 'sep-9999-poll-max-events-cap', + 'sep-9999-poll-stateless-request', + 'sep-9999-poll-errors-are-jsonrpc', + 'sep-9999-poll-invalid-arguments', + 'sep-9999-poll-mode-unsupported', + 'sep-9999-removal-poll-not-found' +] as const; + +const OCCURRENCE_IDS = [ + 'sep-9999-occurrence-event-id', + 'sep-9999-occurrence-name', + 'sep-9999-occurrence-timestamp', + 'sep-9999-occurrence-data', + 'sep-9999-occurrence-cursor-optional', + 'sep-9999-occurrence-meta-ungoverned', + 'sep-9999-occurrence-event-id-from-upstream' +] as const; + +const CURSOR_IDS = [ + 'sep-9999-cursor-opaque', + 'sep-9999-cursor-null-starts-from-now', + 'sep-9999-cursor-absent-equals-null', + 'sep-9999-cursor-consistency', + 'sep-9999-cursor-advances-when-quiet', + 'sep-9999-max-age-ms-floor', + 'sep-9999-max-age-ms-sets-truncated', + 'sep-9999-max-age-ms-ignored-when-cursor-null', + 'sep-9999-replay-ceiling-sets-truncated', + 'sep-9999-truncated-returns-fresh-cursor', + 'sep-9999-truncated-poll-never-an-error', + 'sep-9999-truncated-false-when-no-replay' +] as const; + +const ALL_IDS = [...POLL_IDS, ...OCCURRENCE_IDS, ...CURSOR_IDS]; + +/** Milliseconds of replay to request when probing the `maxAgeMs` floor. */ +const MAX_AGE_PROBE_MS = 300_000; + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, reason, 'SKIPPED', { errorMessage: reason }) + ); +} + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +export class EventsPollScenario implements ClientScenario { + name = 'events-poll'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: \`events/poll\` delivery, the \`EventOccurrence\` shape, and cursor lifecycle. + +**Methods**: \`events/poll\`, plus \`events/list\` to discover a poll-capable event type + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-poll-implemented\` / \`sep-9999-poll-bootstraps-subscription\` — a poll with \`cursor: null\` succeeds with no prior subscribe step +- \`sep-9999-poll-events-array\` / \`sep-9999-poll-response-cursor\` / \`sep-9999-poll-next-poll-ms\` / \`sep-9999-poll-has-more\` — the four response fields +- \`sep-9999-poll-max-events-cap\` — \`maxEvents\` caps the batch and sets \`hasMore\` when more remain +- \`sep-9999-poll-stateless-request\` — two identical polls are answerable without server-side memory of the first +- \`sep-9999-poll-errors-are-jsonrpc\` / \`sep-9999-removal-poll-not-found\` / \`sep-9999-poll-invalid-arguments\` / \`sep-9999-poll-mode-unsupported\` — the error contract +- \`sep-9999-occurrence-*\` — \`eventId\`, \`name\`, \`timestamp\`, \`data\` required; \`cursor\` and \`_meta\` optional +- \`sep-9999-cursor-*\` — opaqueness, \`null\` means start-from-now, absent means \`null\`, consistency, and advancement during quiet periods +- \`sep-9999-max-age-ms-*\` / \`sep-9999-truncated-*\` — bounding replay and signalling a gap + +**Discovery is dynamic**: a server that neither declares the capability nor implements \`events/list\` SKIPs everything; no poll-capable event type reports the poll checks as untestable. Checks that need a delivered event (the \`EventOccurrence\` shape) are untestable against a quiet server rather than passing vacuously.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + return await this.checks(conn); + } finally { + await conn.close(); + } + } + + private async checks(conn: Connection): Promise { + const { declared } = await declaredEventsCapability(conn); + const listed = await eventsListAll(conn); + + if ('error' in listed) { + // Undeclared and unimplemented is the one case that legitimately skips: + // the server simply does not do events. Every other shape is graded, + // including the undeclared-but-serving case events-discovery fails on. + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no poll-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const descriptors = listed.descriptors; + const target = firstSupporting(descriptors, 'poll'); + const name = target ? descriptorName(target) : undefined; + + if (!target || !name) { + return untestableAll( + ALL_IDS, + descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no poll-capable event type could be exercised.' + : 'No event type advertises `poll` delivery, so `events/poll` could not be exercised. Poll is optional per event type.' + ); + } + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties, so the harness cannot construct arguments it is confident the server will accept.` + ); + } + + const checks: ConformanceCheck[] = []; + + // --- The bootstrap poll ---------------------------------------------- + const first = await eventsPoll(conn, { + name, + arguments: args, + cursor: null + }); + if ('error' in first) { + const err = first.error; + checks.push( + eventsCheck( + 'sep-9999-poll-implemented', + '`events/poll` is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.', + 'FAILURE', + { + errorMessage: `Event type \`${name}\` advertises \`poll\` delivery but \`${EVENTS_POLL_METHOD}\` failed: ${err.code} ${err.message}`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-9999-poll-implemented'), + `The bootstrap \`${EVENTS_POLL_METHOD}\` for \`${name}\` failed with ${err.code} ${err.message}.` + ) + ); + return checks; + } + + const r1 = first.result; + checks.push( + eventsCheck( + 'sep-9999-poll-implemented', + '`events/poll` is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.', + 'SUCCESS' + ) + ); + checks.push( + eventsCheck( + 'sep-9999-poll-bootstraps-subscription', + 'No separate subscribe step needed — the first poll with a null cursor bootstraps the subscription.', + 'SUCCESS', + { details: { name } } + ) + ); + + checks.push(...this.responseShapeChecks(r1)); + checks.push(...this.cursorNullChecks(r1)); + + // --- Quiet-period advancement ---------------------------------------- + checks.push(...(await this.quietAdvanceChecks(conn, name, args, r1))); + + // --- maxEvents / hasMore --------------------------------------------- + checks.push(...(await this.maxEventsChecks(conn, name, args))); + + // --- maxAgeMs and truncated ------------------------------------------ + checks.push(...(await this.replayChecks(conn, name, args, r1))); + + // --- EventOccurrence shape ------------------------------------------- + checks.push(...this.occurrenceChecks(r1, target)); + + // --- Error contract --------------------------------------------------- + checks.push(...(await this.errorChecks(conn, descriptors, name, args))); + + return checks; + } + + /** The four response fields, graded off the bootstrap poll. */ + private responseShapeChecks(r: EventsPollResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + + out.push( + Array.isArray(r.events) + ? eventsCheck( + 'sep-9999-poll-events-array', + 'The poll response carries an `events` array. An empty array means nothing happened.', + 'SUCCESS', + { details: { count: occurrences(r).length } } + ) + : eventsCheck( + 'sep-9999-poll-events-array', + 'The poll response carries an `events` array. An empty array means nothing happened.', + 'FAILURE', + { + errorMessage: `\`events\` is ${describeValue(r.events)}, expected an array (empty when nothing happened).`, + details: { events: r.events } + } + ) + ); + + out.push( + isValidCursor(r.cursor) + ? eventsCheck( + 'sep-9999-poll-response-cursor', + 'The poll response carries `cursor` at the response level, the subscription position after this batch.', + 'SUCCESS', + { details: { cursor: r.cursor ?? null } } + ) + : eventsCheck( + 'sep-9999-poll-response-cursor', + 'The poll response carries `cursor` at the response level, the subscription position after this batch.', + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(r.cursor)}, expected a string, null, or absent.`, + details: { cursor: r.cursor } + } + ) + ); + + // nextPollMs is the field the 197c32b4 rename introduced. A server still + // emitting nextPollSeconds is the single most likely failure here, so name + // it rather than reporting a generic absence. + const legacy = 'nextPollSeconds' in r; + if (typeof r.nextPollMs === 'number' && Number.isFinite(r.nextPollMs)) { + out.push( + eventsCheck( + 'sep-9999-poll-next-poll-ms', + '`nextPollMs` allows the server to dynamically adjust polling frequency.', + 'SUCCESS', + { details: { nextPollMs: r.nextPollMs } } + ) + ); + } else { + out.push( + eventsCheck( + 'sep-9999-poll-next-poll-ms', + '`nextPollMs` allows the server to dynamically adjust polling frequency.', + 'WARNING', + { + errorMessage: legacy + ? 'Response carries `nextPollSeconds`, the pre-rename field name. Spec commit `197c32b4` (2026-05-10) renamed the duration fields to `nextPollMs`.' + : `\`nextPollMs\` is ${describeValue(r.nextPollMs)}, expected a number of milliseconds.`, + details: { + nextPollMs: r.nextPollMs, + nextPollSeconds: r.nextPollSeconds + } + } + ) + ); + } + + out.push( + r.hasMore === undefined || typeof r.hasMore === 'boolean' + ? eventsCheck( + 'sep-9999-poll-has-more', + '`hasMore` indicates whether additional events are available beyond the returned batch.', + 'SUCCESS', + { details: { hasMore: r.hasMore ?? false } } + ) + : eventsCheck( + 'sep-9999-poll-has-more', + '`hasMore` indicates whether additional events are available beyond the returned batch.', + 'FAILURE', + { + errorMessage: `\`hasMore\` is ${describeValue(r.hasMore)}, expected a boolean.`, + details: { hasMore: r.hasMore } + } + ) + ); + + // `nextPollMs` is ignored when `hasMore` is true, which is only observable + // on a response that actually sets it. + out.push( + r.hasMore === true + ? eventsCheck( + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + '`nextPollMs` is ignored when `hasMore` is `true`.', + 'SUCCESS', + { details: { hasMore: true, nextPollMs: r.nextPollMs } } + ) + : untestableCheck( + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + 'sep-9999-poll-next-poll-ms-ignored-when-has-more', + '`nextPollMs` is ignored when `hasMore` is `true`.', + 'Server has no backlog, so no response set `hasMore: true` and the interaction between the two fields could not be observed.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** What a `cursor: null` bootstrap poll establishes on its own. */ + private cursorNullChecks(r: EventsPollResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const events = occurrences(r); + + out.push( + events.length === 0 + ? eventsCheck( + 'sep-9999-cursor-null-starts-from-now', + 'Passing `cursor: null` means "start from now." No historical events are replayed.', + 'SUCCESS', + { details: { returned: 0 } } + ) + : eventsCheck( + 'sep-9999-cursor-null-starts-from-now', + 'Passing `cursor: null` means "start from now." No historical events are replayed.', + 'FAILURE', + { + errorMessage: `A poll with \`cursor: null\` returned ${events.length} event(s); "start from now" replays nothing.`, + details: { count: events.length } + } + ) + ); + + out.push( + isValidCursor(r.cursor) + ? eventsCheck( + 'sep-9999-cursor-opaque', + 'Cursors are opaque strings managed by the server, representing a position in the event stream.', + 'SUCCESS', + { + details: { + cursorType: + r.cursor === null || r.cursor === undefined + ? 'null' + : 'string' + } + } + ) + : eventsCheck( + 'sep-9999-cursor-opaque', + 'Cursors are opaque strings managed by the server, representing a position in the event stream.', + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(r.cursor)}; a cursor is an opaque string (or null when the type has no replay).`, + details: { cursor: r.cursor } + } + ) + ); + + return out; + } + + /** + * Poll twice with the cursor from the first response and check the server + * answers the second without being told anything it was not told the first + * time. + * + * This grades three rows at once: the cursor advances (or stays put + * legitimately) during a quiet period, the request is self-contained, and an + * omitted `cursor` field is accepted as `null`. + */ + private async quietAdvanceChecks( + conn: Connection, + name: string, + args: Record, + first: EventsPollResult + ): Promise { + const out: ConformanceCheck[] = []; + const cursor = first.cursor; + const noReplay = cursor === null || cursor === undefined; + + const second = await eventsPoll(conn, { + name, + arguments: args, + ...(noReplay ? {} : { cursor }) + }); + + if ('error' in second) { + const reason = `A follow-up \`${EVENTS_POLL_METHOD}\` carrying the cursor from the first response failed: ${second.error.code} ${second.error.message}`; + out.push( + eventsCheck( + 'sep-9999-poll-stateless-request', + 'Each poll request is self-contained: the server does not need to remember previous poll requests to answer them.', + 'FAILURE', + { errorMessage: reason, details: { code: second.error.code } } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-cursor-advances-when-quiet', + 'sep-9999-cursor-consistency' + ], + reason + ) + ); + out.push( + ...untestableAll( + ['sep-9999-cursor-absent-equals-null'], + reason, + 'FAILURE' + ) + ); + return out; + } + + const r2 = second.result; + + out.push( + eventsCheck( + 'sep-9999-poll-stateless-request', + 'Each poll request is self-contained: the server does not need to remember previous poll requests to answer them.', + 'SUCCESS', + { details: { secondPollAccepted: true } } + ) + ); + + // Cursor advancement during a quiet period. A server with no traffic may + // legitimately return the same cursor — the requirement is that the + // response carries one at all, so the client's persisted position does not + // go stale. An event type with no replay carries null both times, which is + // equally conformant. + out.push( + isValidCursor(r2.cursor) + ? eventsCheck( + 'sep-9999-cursor-advances-when-quiet', + "Every poll response carries `cursor`, including when `events: []`, so the client's persisted cursor advances during quiet periods.", + 'SUCCESS', + { + details: { + first: cursor ?? null, + second: r2.cursor ?? null, + advanced: (r2.cursor ?? null) !== (cursor ?? null) + } + } + ) + : eventsCheck( + 'sep-9999-cursor-advances-when-quiet', + "Every poll response carries `cursor`, including when `events: []`, so the client's persisted cursor advances during quiet periods.", + 'FAILURE', + { + errorMessage: `Follow-up poll returned \`cursor\` as ${describeValue(r2.cursor)}; a quiet poll must still carry a position.`, + details: { cursor: r2.cursor } + } + ) + ); + + // Consistency: a type that returns a cursor once returns one always. + const firstNull = cursor === null || cursor === undefined; + const secondNull = r2.cursor === null || r2.cursor === undefined; + out.push( + firstNull === secondNull + ? eventsCheck( + 'sep-9999-cursor-consistency', + 'An event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.', + 'SUCCESS', + { details: { replaySupported: !firstNull } } + ) + : eventsCheck( + 'sep-9999-cursor-consistency', + 'An event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`.', + 'WARNING', + { + errorMessage: `Event type \`${name}\` returned ${firstNull ? 'null' : 'a cursor'} then ${secondNull ? 'null' : 'a cursor'}; clients branch once at subscribe time on this.`, + details: { first: cursor ?? null, second: r2.cursor ?? null } + } + ) + ); + + // Absent means null: the second poll omitted `cursor` entirely when the + // type has no replay, and the server answered anyway. + out.push( + noReplay + ? eventsCheck( + 'sep-9999-cursor-absent-equals-null', + 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`; a receiver MUST NOT fail because it is missing.', + 'SUCCESS', + { details: { omittedCursorAccepted: true } } + ) + : await this.absentCursorCheck(conn, name, args) + ); + + return out; + } + + /** Probe "absent means null" directly by omitting the field. */ + private async absentCursorCheck( + conn: Connection, + name: string, + args: Record + ): Promise { + const id = 'sep-9999-cursor-absent-equals-null'; + const description = + 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`; a receiver MUST NOT fail because it is missing.'; + const probe = await eventsPoll(conn, { name, arguments: args }); + if ('error' in probe) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll omitting \`cursor\` was rejected with ${probe.error.code} ${probe.error.message}; an absent cursor means "start from now", not a malformed request.`, + details: { code: probe.error.code, message: probe.error.message } + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { returned: occurrences(probe.result).length } + }); + } + + /** `maxEvents` caps the batch; `hasMore` reports whether more remain. */ + private async maxEventsChecks( + conn: Connection, + name: string, + args: Record + ): Promise { + const id = 'sep-9999-poll-max-events-cap'; + const description = + '`maxEvents` is an optional cap on the number of events returned. If more are available, the server returns a partial batch with an intermediate cursor and sets `hasMore: true`.'; + + const probe = await eventsPoll(conn, { + name, + arguments: args, + cursor: null, + maxEvents: 1 + }); + + if ('error' in probe) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll carrying \`maxEvents: 1\` was rejected with ${probe.error.code} ${probe.error.message}; \`maxEvents\` is an optional request field, not an error.`, + details: { code: probe.error.code, message: probe.error.message } + }) + ]; + } + + const returned = occurrences(probe.result).length; + if (returned > 1) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`maxEvents: 1\` returned ${returned} events.`, + details: { returned } + }) + ]; + } + + return [ + eventsCheck(id, description, 'SUCCESS', { + details: { returned, hasMore: probe.result.hasMore ?? false } + }) + ]; + } + + /** + * `maxAgeMs` and `truncated`. + * + * Against a quiet server the floor never advances past the cursor, so the + * two `truncated`-setting rows cannot be driven to true. What is gradeable + * everywhere: the field is accepted, it is ignored when the cursor is null, + * `truncated` is a boolean rather than an error, and when it is true the + * response still carries a fresh cursor. + */ + private async replayChecks( + conn: Connection, + name: string, + args: Record, + bootstrap: EventsPollResult + ): Promise { + const out: ConformanceCheck[] = []; + const noReplay = + bootstrap.cursor === null || bootstrap.cursor === undefined; + + const withMaxAge = await eventsPoll(conn, { + name, + arguments: args, + cursor: bootstrap.cursor ?? null, + maxAgeMs: MAX_AGE_PROBE_MS + }); + + if ('error' in withMaxAge) { + const reason = `A poll carrying \`maxAgeMs\` was rejected with ${withMaxAge.error.code} ${withMaxAge.error.message}.`; + out.push( + eventsCheck( + 'sep-9999-max-age-ms-floor', + 'All three modes accept an optional `maxAgeMs` alongside `cursor`; the server begins replay from whichever is later, the cursor or `now − maxAgeMs`.', + 'FAILURE', + { errorMessage: reason, details: { code: withMaxAge.error.code } } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-max-age-ms-sets-truncated', + 'sep-9999-replay-ceiling-sets-truncated', + 'sep-9999-truncated-returns-fresh-cursor', + 'sep-9999-truncated-poll-never-an-error', + 'sep-9999-truncated-false-when-no-replay', + 'sep-9999-max-age-ms-ignored-when-cursor-null' + ], + reason, + 'WARNING' + ) + ); + return out; + } + + const r = withMaxAge.result; + out.push( + eventsCheck( + 'sep-9999-max-age-ms-floor', + 'All three modes accept an optional `maxAgeMs` alongside `cursor`; the server begins replay from whichever is later, the cursor or `now − maxAgeMs`.', + 'SUCCESS', + { details: { maxAgeMs: MAX_AGE_PROBE_MS } } + ) + ); + + // truncated is a response field, never an error. A server that rejected + // the maxAgeMs poll outright already failed above. + const truncated = r.truncated; + out.push( + truncated === undefined || typeof truncated === 'boolean' + ? eventsCheck( + 'sep-9999-truncated-poll-never-an-error', + 'For poll, `truncated` appears in the result body. Never a JSON-RPC error.', + 'SUCCESS', + { details: { truncated: truncated ?? false } } + ) + : eventsCheck( + 'sep-9999-truncated-poll-never-an-error', + 'For poll, `truncated` appears in the result body. Never a JSON-RPC error.', + 'FAILURE', + { + errorMessage: `\`truncated\` is ${describeValue(truncated)}, expected a boolean.`, + details: { truncated } + } + ) + ); + + // When truncated is true the response must still carry a servable cursor. + out.push( + truncated === true + ? isValidCursor(r.cursor) && r.cursor !== null && r.cursor !== undefined + ? eventsCheck( + 'sep-9999-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'SUCCESS', + { details: { cursor: r.cursor } } + ) + : eventsCheck( + 'sep-9999-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'FAILURE', + { + errorMessage: `\`truncated: true\` was returned with \`cursor\` as ${describeValue(r.cursor)}; the client has no fresh position to persist.`, + details: { cursor: r.cursor } + } + ) + : untestableCheck( + 'sep-9999-truncated-returns-fresh-cursor', + 'sep-9999-truncated-returns-fresh-cursor', + 'The server resets to a position it can serve from and returns that position as the fresh `cursor` alongside `truncated: true`.', + 'No probe produced `truncated: true`, so the fresh-cursor obligation could not be observed. Driving it requires a stale cursor the harness cannot mint against an opaque cursor space.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + for (const id of [ + 'sep-9999-max-age-ms-sets-truncated', + 'sep-9999-replay-ceiling-sets-truncated' + ]) { + out.push( + untestableCheck( + id, + id, + id, + "Requires a cursor older than the `maxAgeMs` floor or the server's replay ceiling. Cursors are opaque, so the harness cannot mint a stale one, and a quiet fixture has no history to fall out of.", + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + out.push( + noReplay + ? truncated === true + ? eventsCheck( + 'sep-9999-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'WARNING', + { + errorMessage: `Event type \`${name}\` returns \`cursor: null\` (no replay) but set \`truncated: true\`; there is no position to have advanced past.`, + details: { truncated } + } + ) + : eventsCheck( + 'sep-9999-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'SUCCESS', + { details: { truncated: truncated ?? false } } + ) + : eventsCheck( + 'sep-9999-truncated-false-when-no-replay', + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.', + 'SKIPPED', + { + errorMessage: `Event type \`${name}\` supports replay, so this rule does not apply to it.` + } + ) + ); + + // maxAgeMs is ignored when cursor is null. Observable as "the server does + // not replay history in response to it". + const nullCursorMaxAge = await eventsPoll(conn, { + name, + arguments: args, + cursor: null, + maxAgeMs: MAX_AGE_PROBE_MS + }); + const id = 'sep-9999-max-age-ms-ignored-when-cursor-null'; + const description = + '`maxAgeMs` is ignored when `cursor` is `null` (null already means "now").'; + if ('error' in nullCursorMaxAge) { + out.push( + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`cursor: null\` and \`maxAgeMs\` was rejected with ${nullCursorMaxAge.error.code} ${nullCursorMaxAge.error.message}.`, + details: { code: nullCursorMaxAge.error.code } + }) + ); + } else { + const replayed = occurrences(nullCursorMaxAge.result).length; + out.push( + replayed === 0 + ? eventsCheck(id, description, 'SUCCESS', { details: { replayed } }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll with \`cursor: null\` and \`maxAgeMs: ${MAX_AGE_PROBE_MS}\` replayed ${replayed} event(s); \`maxAgeMs\` is ignored when the cursor is null.`, + details: { replayed } + }) + ); + } + + return out; + } + + /** + * The `EventOccurrence` field contract. + * + * Only gradeable against a response that actually carried an event. A quiet + * fixture reports the whole group as untestable rather than passing an empty + * array through seven shape checks, which would read as green. + */ + private occurrenceChecks( + r: EventsPollResult, + descriptor: EventDescriptor + ): ConformanceCheck[] { + const events = occurrences(r); + if (events.length === 0) { + return untestableAll( + OCCURRENCE_IDS, + 'No poll returned an event, so the `EventOccurrence` shape could not be validated. The fixture needs a diagnostic event type that emits on demand.' + ); + } + + const out: ConformanceCheck[] = []; + const label = (e: EventOccurrence, i: number) => + typeof e.eventId === 'string' ? `\`${e.eventId}\`` : `events[${i}]`; + + const field = ( + id: string, + description: string, + severity: 'FAILURE' | 'WARNING', + predicate: (e: EventOccurrence) => string | undefined + ) => { + for (const [i, e] of events.entries()) { + const problem = predicate(e); + if (problem) { + out.push( + eventsCheck(id, description, severity, { + errorMessage: `${label(e, i)}: ${problem}`, + details: { occurrence: e } + }) + ); + return; + } + } + out.push( + eventsCheck(id, description, 'SUCCESS', { + details: { occurrencesChecked: events.length } + }) + ); + }; + + field( + 'sep-9999-occurrence-event-id', + '`eventId` (string) is required on every `EventOccurrence`: a stable identifier for deduplication.', + 'FAILURE', + (e) => + typeof e.eventId === 'string' && e.eventId.length > 0 + ? undefined + : `\`eventId\` is ${describeValue(e.eventId)}, expected a non-empty string.` + ); + + field( + 'sep-9999-occurrence-name', + '`name` (string) is required on every `EventOccurrence`: the event type name.', + 'FAILURE', + (e) => + typeof e.name === 'string' && e.name === descriptorName(descriptor) + ? undefined + : `\`name\` is ${JSON.stringify(e.name)}, expected \`${descriptorName(descriptor)}\`.` + ); + + field( + 'sep-9999-occurrence-timestamp', + '`timestamp` (string, ISO 8601) is required on every `EventOccurrence`: when the event occurred.', + 'FAILURE', + (e) => + isIso8601(e.timestamp) + ? undefined + : `\`timestamp\` is ${JSON.stringify(e.timestamp)}, expected an ISO 8601 instant.` + ); + + field( + 'sep-9999-occurrence-data', + "`data` (object) is required on every `EventOccurrence`: payload conforming to the event type's `payloadSchema`.", + 'FAILURE', + (e) => + isObject(e.data) + ? undefined + : `\`data\` is ${describeValue(e.data)}, expected an object.` + ); + + field( + 'sep-9999-occurrence-cursor-optional', + '`cursor` on an `EventOccurrence` is optional; poll carries the cursor at the response level.', + 'FAILURE', + (e) => + isValidCursor(e.cursor) + ? undefined + : `\`cursor\` is ${describeValue(e.cursor)}, expected a string, null, or absent.` + ); + + field( + 'sep-9999-occurrence-meta-ungoverned', + '`_meta` is reserved for protocol/extension metadata and is not governed by `payloadSchema`.', + 'WARNING', + (e) => + e._meta === undefined || isObject(e._meta) + ? undefined + : `\`_meta\` is ${describeValue(e._meta)}, expected an object when present.` + ); + + // Whether an eventId came from upstream is not observable from one run; + // what is observable is that ids are distinct within a batch, which a + // per-delivery counter would violate. + const ids = events + .map((e) => e.eventId) + .filter((v): v is string => typeof v === 'string'); + out.push( + new Set(ids).size === ids.length + ? eventsCheck( + 'sep-9999-occurrence-event-id-from-upstream', + "The server SHOULD use the upstream's stable event identifier as `eventId` so the same upstream event carries the same id across delivery paths.", + 'SUCCESS', + { details: { distinct: new Set(ids).size, total: ids.length } } + ) + : eventsCheck( + 'sep-9999-occurrence-event-id-from-upstream', + "The server SHOULD use the upstream's stable event identifier as `eventId` so the same upstream event carries the same id across delivery paths.", + 'WARNING', + { + errorMessage: + 'A single batch repeated an `eventId`, so the value cannot be an upstream-stable identifier and client-side dedup would drop distinct events.', + details: { ids } + } + ) + ); + + return out; + } + + /** The poll error contract: unknown name, bad arguments, unsupported mode. */ + private async errorChecks( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + args: Record + ): Promise { + const out: ConformanceCheck[] = []; + + // Unknown name. + const unknown = unknownEventName(); + const probe = await eventsPoll(conn, { + name: unknown, + arguments: {}, + cursor: null + }); + const notFoundDesc = + 'A poll against a name the server does not serve returns `-32011 NotFound`.'; + const jsonRpcDesc = + 'Errors are returned as a standard JSON-RPC error response for the request; there is no partial-success model.'; + + if ('error' in probe) { + out.push( + eventsCheck( + 'sep-9999-poll-errors-are-jsonrpc', + jsonRpcDesc, + 'SUCCESS', + { + details: { code: probe.error.code } + } + ) + ); + out.push( + probe.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-9999-removal-poll-not-found', + notFoundDesc, + 'SUCCESS', + { + details: { code: probe.error.code, probedName: unknown } + } + ) + : eventsCheck( + 'sep-9999-removal-poll-not-found', + notFoundDesc, + 'FAILURE', + { + errorMessage: `Unknown event name answered ${probe.error.code}, expected ${EVENTS_NOT_FOUND} NotFound.`, + details: { + code: probe.error.code, + message: probe.error.message + } + } + ) + ); + } else { + const msg = `A poll for unknown event name \`${unknown}\` returned a result instead of an error.`; + out.push( + eventsCheck( + 'sep-9999-poll-errors-are-jsonrpc', + jsonRpcDesc, + 'FAILURE', + { + errorMessage: msg, + details: { result: probe.result } + } + ) + ); + out.push( + eventsCheck( + 'sep-9999-removal-poll-not-found', + notFoundDesc, + 'FAILURE', + { + errorMessage: msg + } + ) + ); + } + + // One subscription per request: `name` identifies it, so a poll without + // one is not a request the server can answer. + const noName = await eventsPoll(conn, { arguments: {}, cursor: null }); + const oneSubDesc = + 'Each `events/poll` request carries one subscription, identified by `name`.'; + out.push( + 'error' in noName + ? noName.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck( + 'sep-9999-poll-one-subscription-per-request', + oneSubDesc, + 'SUCCESS', + { details: { code: noName.error.code } } + ) + : eventsCheck( + 'sep-9999-poll-one-subscription-per-request', + oneSubDesc, + 'WARNING', + { + errorMessage: `A poll omitting \`name\` answered ${noName.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + code: noName.error.code, + message: noName.error.message + } + } + ) + : eventsCheck( + 'sep-9999-poll-one-subscription-per-request', + oneSubDesc, + 'FAILURE', + { + errorMessage: + 'A poll omitting `name` returned a result; each request carries exactly one subscription and `name` is what identifies it.', + details: { result: noName.result } + } + ) + ); + + // Invalid arguments. Only probeable when the schema constrains something. + out.push(await this.invalidArgumentsCheck(conn, descriptors, name, args)); + + // A delivery mode the event type does not offer. + out.push(await this.unsupportedModeCheck(conn, descriptors)); + + return out; + } + + /** + * Send arguments the descriptor's `inputSchema` cannot accept. + * + * Only constructible when the schema declares a typed property: a fully open + * schema has no invalid value to send, and inventing one would grade the + * server on a rule the schema never stated. + */ + private async invalidArgumentsCheck( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + _args: Record + ): Promise { + const id = 'sep-9999-poll-invalid-arguments'; + const description = + "A poll whose `arguments` do not match the event's `inputSchema` returns `-32602 InvalidParams`."; + + const target = descriptors.find((d) => descriptorName(d) === name); + const schema = target?.inputSchema; + const props = + isObject(schema) && isObject(schema.properties) + ? schema.properties + : undefined; + const typed = props + ? Object.entries(props).find( + ([, v]) => + isObject(v) && + typeof v.type === 'string' && + ['string', 'boolean', 'integer', 'number'].includes(v.type) + ) + : undefined; + + if (!typed) { + return untestableCheck( + id, + id, + description, + `Event type \`${name}\` declares no typed \`inputSchema\` property, so no argument value can be known-invalid against it.`, + [EVENTS_SPEC_REF], + 'FAILURE' + ); + } + + const [prop, spec] = typed; + const propType = (spec as Record).type as string; + // A value of the wrong JSON type for the declared one. + const wrongValue = propType === 'string' ? 12345 : 'not-a-valid-value'; + + const probe = await eventsPoll(conn, { + name, + arguments: { [prop]: wrongValue }, + cursor: null + }); + + if (!('error' in probe)) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll sending \`${prop}: ${JSON.stringify(wrongValue)}\` against a declared \`${propType}\` returned a result instead of ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { property: prop, declaredType: propType, sent: wrongValue } + }); + } + + return probe.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck(id, description, 'SUCCESS', { + details: { property: prop, declaredType: propType } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Arguments violating \`inputSchema\` answered ${probe.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + property: prop, + declaredType: propType, + code: probe.error.code, + message: probe.error.message + } + }); + } + + /** Poll an event type whose `delivery` omits `poll`. */ + private async unsupportedModeCheck( + conn: Connection, + descriptors: EventDescriptor[] + ): Promise { + const id = 'sep-9999-poll-mode-unsupported'; + const description = + 'A poll against an event type whose `delivery` does not list `poll` returns `-32014 Unsupported`.'; + + const nonPoll = descriptors.find( + (d) => + descriptorName(d) !== undefined && !deliveryModes(d).includes('poll') + ); + if (!nonPoll) { + return untestableCheck( + id, + id, + description, + 'Every event type the server offers advertises `poll` delivery, so there is no event type to probe the unsupported-mode path with.', + [EVENTS_SPEC_REF], + 'FAILURE' + ); + } + + const name = descriptorName(nonPoll)!; + const probe = await eventsPoll(conn, { + name, + arguments: minimalArguments(nonPoll) ?? {}, + cursor: null + }); + + if (!('error' in probe)) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Event type \`${name}\` does not advertise \`poll\` delivery (\`delivery\` is ${JSON.stringify(nonPoll.delivery)}) but \`${EVENTS_POLL_METHOD}\` returned a result.`, + details: { name, delivery: deliveryModes(nonPoll) } + }); + } + + return probe.error.code === EVENTS_UNSUPPORTED + ? eventsCheck(id, description, 'SUCCESS', { + details: { name, code: probe.error.code, data: probe.error.data } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Polling \`${name}\`, which does not offer poll delivery, answered ${probe.error.code}, expected ${EVENTS_UNSUPPORTED} Unsupported.`, + details: { + name, + delivery: deliveryModes(nonPoll), + code: probe.error.code, + message: probe.error.message + } + }); + } +} + +/** Exported for the negative tests, which assert the full emitted set. */ +export const EVENTS_POLL_CHECK_IDS = ALL_IDS; diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml new file mode 100644 index 00000000..15497836 --- /dev/null +++ b/src/seps/sep-9999.yaml @@ -0,0 +1,537 @@ +# spec_source: modelcontextprotocol/experimental-ext-triggers-events@main docs/design-sketch-proposal.md +# extracted: 2026-09-15 +# +# ############################################################################ +# # 9999 IS A PLACEHOLDER. MCP Events has no SEP number yet. # +# # Renaming it is a release blocker before this file is upstreamed. # +# ############################################################################ +# +# SEP numbers are PR numbers in modelcontextprotocol/modelcontextprotocol, +# and no events PR has been opened there — the work lives in the separate +# experimental-ext-triggers-events repository. Both ends enforce a number: +# src/traceability/index.ts:174 filters the directory on `/^sep-\d+\.yaml$/` +# and line 41 filters emitted check IDs on `/^sep-\d+-/`, so a file named +# `events.yaml` is silently dropped from the manifest rather than rejected; +# src/new-sep/index.ts:163 rejects any argument that is not a positive +# integer from the other side. There is no honest name that works. +# +# 9999 is far from the live range (~2663), so a collision is implausible. +# The rename is mechanical and wide: this filename, the `sep:` field below, +# every `sep-9999-` id here, and the same ids in +# src/scenarios/server/events/. +# +# This must not merge under the placeholder. plan.modelcontextprotocol.io +# reads the manifest built from `main`, so a draft PR is invisible to it, but +# a merge would publish SEP 9999 as though it were real and a later rename +# would then have to retract it. The rename is therefore a merge blocker, not +# a follow-up. +# +# The reservation question is open with the WG in #triggers-events-wg. Peter +# is the person to ask and is away the week of 2026-09-14 for AGNTCon, so the +# answer is not expected before the scenarios land. +# +# provenance: extracted against the merged design sketch on `main`, which +# landed 2026-09-08. Before that merge the document was a moving PR head and +# not worth scoring against. Two spec commits land inside this extraction: +# `197c32b4` (2026-05-10) renamed the duration fields to `nextPollMs` and +# `maxAgeMs`, and `28ec35e9` (2026-09-04) added event-type removal and +# breaking-change termination. +# +# The sketch is a design document rather than a spec diff, so there is no +# `docs/specification/draft/*.mdx` to point `new-sep` at — `specPathToUrl` +# (src/new-sep/index.ts:19) hard-requires that prefix, so `--spec-path` does +# not help either. The file was scaffolded with `--spec-url`, which +# short-circuits the GitHub lookup. There are no section anchors to cite, so +# `spec_url` names the document itself and rows carry no per-row `url`. +# Expect one more pass once the text becomes a spec PR and gains anchors. +# +# coverage: 131 declared checks plus 30 excluded rows. +# +# A keyword sweep of the source finds 144 RFC 2119 keyword occurrences across +# 119 sentences: 34 MUST, 9 MUST NOT, 3 REQUIRED, 56 SHOULD, 7 SHOULD NOT +# (109 normative), plus 34 MAY and 1 OPTIONAL. Counting MUST NOT inside MUST +# and SHOULD NOT inside SHOULD, as the handoff table does, the same sweep +# reads as 43 / 3 / 63; the two counts agree. +# +# Only 46 of the declared rows quote a sentence carrying a keyword. The other +# 85 are shape requirements the document states declaratively — the +# `EventOccurrence` field table, the error-code table, the `events/list` +# descriptor fields, the response payloads. They are normative and directly +# testable, they are simply not written with a keyword, which is why the row +# count exceeds the sentence count in the sweep. One keyword sentence also +# frequently carries several distinct obligations (the Standard Webhooks +# signature sentence, the subscription-key sentence), which pushes the same +# way. +# +# severity rule, since the usual "follow the keyword" mapping is undefined +# for a shape row: MUST / MUST NOT / REQUIRED -> FAILURE (30 rows), SHOULD / +# SHOULD NOT -> WARNING (16 rows). For a shape row, a field the document +# marks Required in a schema table, or a wire fact a client cannot work +# without (an error code, a method existing at all), is FAILURE; everything +# else drawn from an example payload is WARNING. +# +# Pure MAY and OPTIONAL sentences get no row at all, per the handoff +# disposition table. The five that initially got one are demoted to +# `excluded:` at the foot of this file rather than deleted, so the sweep +# stays auditable against the document. +# +# excluded rather than declared: client-SDK and host obligations (poll floor, +# payload sanitization, policy evaluation), receiver obligations the harness +# implements rather than grades, server-SDK implementation guidance from the +# two "SDK Guidance" sections, and the WAF/deployment profile advisories. +# None is a server wire obligation, so declaring them would inflate the +# denominator with rows no server-side scenario could ever emit. +# +# deliberately not declared here: pagination semantics for `events/list`. The +# sketch defers them ("same semantics as tools/list etc."), so the obligation +# is the base protocol's and belongs to the core pagination scenarios. +# `sep-9999-list-pagination` declares only the part this document adds, which +# is that `events/list` participates in that scheme at all. +# +# An `events/poll` against an unknown name is stated twice, once under Error +# Codes and once as the poll consequence of an event type having been +# removed. Only `sep-9999-removal-poll-not-found` is declared for it; a +# second row would double-count one probe. +# +# `## Key Design Decisions`, `## Open Questions` and `## What Is NOT in v1` +# are rationale rather than requirements and produce no rows. The design +# table restates the webhook secret, TTL and TLS rules already declared above +# it. +# +# backing_scenarios: server ClientScenarios under src/scenarios/server/events/ +# emit the check IDs below (a row is "tested" once a scenario emits its ID; +# see src/traceability/). Phase 1 ships two of the five and emits 45 of the +# 131 rows. The remaining 86 report as untested until the later scenarios +# land, which is the manifest working as intended rather than a gap here: +# discovery.ts (events-discovery), 12 rows — the two capability rows, the +# two sep-9999-list-* rows, the six sep-9999-descriptor-* rows, and +# sep-9999-error-not-found plus sep-9999-error-server-range, both graded +# off one unknown-name probe. +# poll.ts (events-poll), 33 rows — the sep-9999-poll-* rows, the +# sep-9999-occurrence-* rows, the sep-9999-cursor-* / +# sep-9999-max-age-* / sep-9999-truncated-* rows reachable through poll, +# and sep-9999-removal-poll-not-found, which is the poll leg of the +# removal rules. +# +# The other four sep-9999-removal-* rows and the remaining sep-9999-error-* +# codes need a server that can be made to remove an event type mid-run, or a +# delivery mode this phase does not drive, so they wait on the push and +# webhook scenarios. +# Still to come, and reported untested in the manifest until they land: +# push.ts (events-push) — the sep-9999-stream-* rows. +# webhook.ts (events-webhook) — the sep-9999-subscribe-*, sep-9999-ttl-* +# and sep-9999-unsubscribe-* rows. +# webhook-delivery.ts (events-webhook-delivery) — the sep-9999-delivery-*, +# sep-9999-verification-*, sep-9999-ssrf-* and sep-9999-envelope-* rows. +# These need a callback URL the server under test can reach over https, +# which localhost cannot satisfy: the SSRF rules declared below require a +# conformant server to refuse it. +# +# measured against mcpkit, 2026-09-15, examples/events/kitchen-sink at main: +# events-discovery 8/11, events-poll 18/28. Six divergences, each a real gap +# rather than something the suite should accommodate. Do not soften these +# rows to make mcpkit pass; catching them is the point. +# +# sep-9999-capability-events-object — the server answers `events/list` but +# declares no `capabilities.events`, so a client that reads capabilities +# before calling never finds the surface. Not previously tracked. This is +# also why the scenario asks before it skips: a plain "undeclared optional +# capability" SKIP reports this server as a clean run. +# sep-9999-poll-next-poll-ms — `nextPollSeconds` persists (events.go:504) +# after `197c32b4` renamed it, and `maxAge` is still in seconds +# (events.go:544). mcpkit's own wire_shape_test.go:44 asserts the +# pre-rename name. Gap G31. +# sep-9999-descriptor-delivery-subset — `events.topology` advertises an +# empty `delivery`, where the document requires a non-empty subset of +# poll/push/webhook. That source is the substitute for the missing +# `notifications/events/list_changed`. Gap G34. +# sep-9999-descriptor-input-schema — no descriptor carries `inputSchema`, +# which also makes `sep-9999-poll-invalid-arguments` untestable, since +# there is no declared constraint left to violate. Not previously tracked. +# sep-9999-poll-events-array — `events` is omitted rather than returned as +# an empty array when nothing happened. Not previously tracked. +# sep-9999-poll-mode-unsupported — `events/poll` answers for an event type +# whose `delivery` does not list `poll`, where the document requires +# `-32014 Unsupported`. Not previously tracked. +# +# The removal and termination rules (gap G30) are declared here but not yet +# exercised: `RemoveSource` (registry.go:182) fires topology only and +# `UnsupportedData` (errors.go:90) has no `reason` field, but driving that +# path needs a server that can drop an event type mid-run, which waits on +# the push scenario. +sep: 9999 +spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md +requirements: + # === Capability Declaration === + - check: sep-9999-capability-events-object + text: 'Servers advertise event support in their capabilities: `{"capabilities": {"events": {"listChanged": true}}}`.' + - check: sep-9999-capability-list-changed-flag + text: 'The `listChanged` flag under `capabilities.events` advertises that the server sends `notifications/events/list_changed`.' + + # === Listing Available Events === + - check: sep-9999-list-implemented + text: '`events/list` returns `{events: [...], nextCursor}`, where each entry describes one event type.' + - check: sep-9999-list-pagination + text: '`nextCursor` is "present when more pages are available; same semantics as tools/list etc."' + - check: sep-9999-descriptor-name + text: 'Each event descriptor carries a `name` identifying the event type.' + - check: sep-9999-descriptor-description + text: 'Each event descriptor carries a `description` of when the event fires.' + - check: sep-9999-descriptor-delivery-subset + text: '`delivery` lists the delivery modes this event type supports — any non-empty subset of `"poll"`, `"push"`, `"webhook"`. No mode is mandatory.' + - check: sep-9999-descriptor-input-schema + text: '`inputSchema` is a JSON Schema describing valid subscription arguments — these may include filters (which narrow the event stream), transforms (which modify payloads), or other server-defined configuration.' + - check: sep-9999-descriptor-payload-schema + text: '`payloadSchema` describes the shape of `data` in delivered events.' + - check: sep-9999-descriptor-meta + text: '`_meta` on an event descriptor is optional; same semantics as on Tool/Resource/Prompt.' + - check: sep-9999-schema-evolution-additive + text: "Servers SHOULD evolve an event type's `inputSchema` and `payloadSchema` additively for the lifetime of its `name`: new optional fields MAY be added; existing fields SHOULD NOT be removed, renamed, or retyped; enums SHOULD NOT be narrowed; and `inputSchema` SHOULD NOT be tightened such that previously accepted `arguments` become invalid." + - check: sep-9999-breaking-change-new-name + text: 'A breaking change SHOULD instead be published under a new event name, served alongside the old one for a migration period, after which the old name is removed.' + + # === Dynamic Event Types === + - check: sep-9999-list-changed-notification + text: 'If the set of available event types, or the descriptor of any of them (`description`, `delivery`, `inputSchema`, `payloadSchema`), changes at runtime, the server sends a `notifications/events/list_changed` notification.' + + # === Event Type Removal and Breaking Changes (spec commit 28ec35e9) === + - check: sep-9999-removal-terminates-subscriptions + text: "The server SHOULD end them using each mode's termination signal: `notifications/events/terminated` on push streams and a `terminated` envelope to webhook subscriptions." + - check: sep-9999-removal-error-not-found + text: 'The `error` is `-32011 NotFound` with `data: {"kind": "event"}` when the type was removed.' + - check: sep-9999-removal-error-schema-changed + text: 'The `error` is `-32014 Unsupported` with `data: {"feature": "payloadSchema" | "inputSchema", "reason": "schema_changed"}` when it was changed in place — not `-32012 Forbidden`, since the principal''s access is unchanged.' + - check: sep-9999-removal-additive-no-terminate + text: 'Purely additive changes MUST NOT terminate subscriptions.' + - check: sep-9999-removal-poll-not-found + text: 'Poll holds no server-side subscription to terminate: a poll against a removed name already returns `-32011 NotFound`.' + + # === Error Codes === + - check: sep-9999-error-invalid-params + text: "`-32602 InvalidParams` — request is statically invalid: arguments don't match the event's inputSchema, the callback `delivery.url` is malformed or non-`https`, or `delivery.secret` is not a valid `whsec_` value." + - check: sep-9999-error-not-found + text: '`-32011 NotFound` — a referenced entity does not exist: an unknown event name, or no subscription matching the key on `events/unsubscribe`.' + - check: sep-9999-error-forbidden + text: '`-32012 Forbidden` — the authenticated principal is not permitted for this event/arguments combination, or its access was revoked.' + - check: sep-9999-error-resource-exhausted + text: '`-32013 ResourceExhausted` — a server-imposed limit or quota was reached. `data.limit` names it (e.g. `"subscriptions"`).' + - check: sep-9999-error-unsupported + text: '`-32014 Unsupported` — the request is well-formed but a requested capability or option is not supported here, e.g. a delivery mode the event type does not offer. `data` identifies it (e.g. `{"feature": "deliveryMode", "value": "push"}`).' + - check: sep-9999-error-callback-endpoint-error + text: '`-32015 CallbackEndpointError` — a client-supplied callback endpoint failed verification or could not be reached (webhook mode only). `data.reason` is one of the `lastError` categories.' + - check: sep-9999-error-server-range + text: 'These are general-purpose codes carried in the JSON-RPC implementation-defined server range `[-32000, -32099]`.' + + # === Poll-Based Delivery === + - check: sep-9999-poll-implemented + text: 'Poll (`events/poll`) is request/response: client sends `{name, arguments, cursor}`, gets back `{events[], cursor, nextPollMs}`.' + - check: sep-9999-poll-one-subscription-per-request + text: 'Each `events/poll` request carries one subscription. A client with multiple subscriptions runs one poll loop per subscription.' + - check: sep-9999-poll-bootstraps-subscription + text: 'No separate subscribe step needed — the first poll with a null cursor bootstraps the subscription. Server holds no protocol-required state.' + - check: sep-9999-poll-events-array + text: 'The poll response carries an `events` array of `EventOccurrence` entries. Empty `events` array means nothing happened — this is the common case and should be cheap.' + - check: sep-9999-poll-response-cursor + text: 'The poll response carries `cursor` at the response level, the subscription position after this batch.' + - check: sep-9999-poll-next-poll-ms + text: '`nextPollMs` allows the server to dynamically adjust polling frequency (e.g., back off when rate-limited upstream, speed up when activity is detected).' + - check: sep-9999-poll-next-poll-ms-ignored-when-has-more + text: '`nextPollMs` is ignored when `hasMore` is `true`.' + - check: sep-9999-poll-has-more + text: '`hasMore` indicates whether additional events are available beyond the returned batch. When `true`, the client should poll again immediately with the updated cursor. When `false`, the client waits `nextPollMs` before the next poll.' + - check: sep-9999-poll-max-events-cap + text: '`maxEvents` is an optional cap on the number of events returned. If more events are available than the limit, the server returns a partial batch with an intermediate cursor and sets `hasMore: true`. If omitted, the server uses its own default limit.' + - check: sep-9999-poll-stateless-request + text: 'Each poll request is self-contained: the client provides the event name, arguments, and cursor. The server does not need to "remember" previous poll requests to answer them.' + - check: sep-9999-poll-errors-are-jsonrpc + text: 'Errors (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`) are returned as a standard JSON-RPC error response for the request — there is no partial-success model since each request carries one subscription.' + - check: sep-9999-poll-invalid-arguments + text: "A poll whose `arguments` do not match the event's `inputSchema` returns `-32602 InvalidParams`." + - check: sep-9999-poll-mode-unsupported + text: 'A poll against an event type whose `delivery` does not list `"poll"` returns `-32014 Unsupported` identifying the unsupported delivery mode.' + + # === EventOccurrence schema === + - check: sep-9999-occurrence-event-id + text: '`eventId` (string) is required on every `EventOccurrence`: a stable identifier for deduplication.' + - check: sep-9999-occurrence-name + text: '`name` (string) is required on every `EventOccurrence`: the event type name.' + - check: sep-9999-occurrence-timestamp + text: '`timestamp` (string, ISO 8601) is required on every `EventOccurrence`: when the event occurred.' + - check: sep-9999-occurrence-data + text: "`data` (object) is required on every `EventOccurrence`: payload conforming to the event type's `payloadSchema`." + - check: sep-9999-occurrence-cursor-optional + text: '`cursor` (string | null) is not required: subscription position after this event (push/webhook only; poll carries cursor at the response level).' + - check: sep-9999-occurrence-meta-ungoverned + text: '`_meta` is reserved for protocol/extension metadata, consistent with `_meta` on other MCP types. Not governed by `payloadSchema`.' + - check: sep-9999-occurrence-event-id-from-upstream + text: '`eventId` is server-assigned: when the upstream source provides a stable event identifier, the server SHOULD use that value as `eventId` so that the same upstream event surfaced via multiple paths carries the same `eventId` and dedup works.' + + # === Cursor Lifecycle === + - check: sep-9999-cursor-opaque + text: 'Cursors are opaque strings managed by the server. They represent a position in the event stream. `cursor` is opaque to the client.' + - check: sep-9999-cursor-null-starts-from-now + text: 'Passing `cursor: null` in any mode means "start from now." The server returns a fresh cursor representing the current position (or `null` if the event type does not support replay). No historical events are replayed.' + - check: sep-9999-cursor-absent-equals-null + text: 'An absent `cursor` field MUST be treated identically to an explicit `cursor: null`: a sender MAY omit the field instead of writing `null`, and a receiver MUST NOT fail because it is missing.' + - check: sep-9999-cursor-consistency + text: 'Servers SHOULD be consistent: an event type that ever returns a non-null cursor SHOULD always do so, and one that returns `null` SHOULD always return `null`, so clients can branch once at subscribe time rather than per delivery.' + - check: sep-9999-cursor-advances-when-quiet + text: "The client's persisted cursor must advance during quiet periods. Poll covers this inherently (every response carries `cursor`, including when `events: []`)." + - check: sep-9999-max-age-ms-floor + text: 'All three modes accept an optional `maxAgeMs` (integer milliseconds) alongside `cursor`. When present, the server begins replay from whichever is later: the supplied `cursor`, or `now − maxAgeMs`.' + - check: sep-9999-max-age-ms-sets-truncated + text: 'If the floor advances past the cursor, the server SHOULD set `truncated: true` on the first response (poll result / `notifications/events/active` / webhook subscribe response) so the client knows older events were skipped.' + - check: sep-9999-max-age-ms-ignored-when-cursor-null + text: '`maxAgeMs` is ignored when `cursor` is `null` (null already means "now") and for event types that do not support replay.' + - check: sep-9999-replay-ceiling-sets-truncated + text: 'Servers MAY also apply their own replay ceiling independent of `maxAgeMs` and MUST signal it via `truncated: true` when they do.' + - check: sep-9999-truncated-returns-fresh-cursor + text: 'In all cases the server resets to a position it can serve from, returns that position as the fresh `cursor` alongside `truncated: true`, and continues — the client never reconnects or re-subscribes in response.' + - check: sep-9999-truncated-poll-never-an-error + text: 'For poll, `truncated` appears in the result body: `{events:[], cursor:, truncated:true, hasMore, nextPollMs}`. Never a JSON-RPC error.' + - check: sep-9999-truncated-false-when-no-replay + text: 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.' + + # === Push-Based Delivery === + - check: sep-9999-stream-implemented + text: 'Push delivery uses a long-lived `events/stream` request — one per subscription. The request is a standard JSON-RPC request with an `id`, which enables cancellation via `notifications/cancelled` and is echoed in every notification for routing.' + - check: sep-9999-stream-error-before-open + text: 'If the subscription is invalid (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`), the server responds immediately with a JSON-RPC error and no stream is opened.' + - check: sep-9999-stream-active-confirmation + text: 'Otherwise the server confirms the subscription with `notifications/events/active {cursor, truncated, _meta.subscriptionId}` and delivers events as notifications.' + - check: sep-9999-stream-subscription-id-meta + text: 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]` so a client with multiple concurrent streams can route notifications to the correct stream.' + - check: sep-9999-stream-event-notification + text: 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.' + - check: sep-9999-stream-error-is-recoverable + text: '`notifications/events/error` reports a recoverable failure (e.g., a single upstream fetch failed); the subscription remains active and the server retries and resumes.' + - check: sep-9999-stream-terminated-ends-subscription + text: 'Only `notifications/events/terminated` ends the subscription.' + - check: sep-9999-stream-gap-resends-active + text: "A gap (e.g., the cursor fell outside the upstream's retention window) is not an error — the server sends a fresh `notifications/events/active {cursor:, truncated:true, _meta.subscriptionId}` and continues delivering." + - check: sep-9999-stream-heartbeat-required + text: 'The server MUST send periodic keepalive messages on the push stream so the client can distinguish "nothing to send" from "connection is dead."' + - check: sep-9999-stream-heartbeat-carries-cursor + text: "The heartbeat is `notifications/events/heartbeat` — `cursor` is the position the server has checked up to, so the client's persisted cursor advances even when no events match; it is `null` for event types that do not support replay." + - check: sep-9999-stream-heartbeat-interval + text: 'The server SHOULD send a heartbeat at least every 30 seconds.' + - check: sep-9999-stream-heartbeat-not-sse-comment + text: 'On Streamable HTTP this is sent as an SSE `data:` frame; the SSE comment form (`: keepalive`) is not used since it cannot carry cursor state.' + - check: sep-9999-stream-final-result-shape + text: 'The `StreamEventsResult` is an empty typed result (`{"_meta": {}}`). It carries no information — it satisfies JSON-RPC''s requirement that every request gets a response.' + - check: sep-9999-stream-final-result-timing + text: 'It is sent whenever the server can write a final frame: on Streamable HTTP only when the server initiates the close; on stdio the server MAY send it, and clients MUST NOT depend on receiving it.' + - check: sep-9999-stream-cancel-stops-delivery + text: 'In both cases, the server MUST stop delivering events and release any associated resources.' + - check: sep-9999-stream-exempt-from-concurrency-cap + text: 'Server SDKs MUST exempt `events/stream` from any general request-concurrency cap, since each push subscription is a long-lived request that never completes until cancelled.' + - check: sep-9999-stream-carries-only-event-notifications + text: "The `events/stream` response carries only `notifications/events/*` messages. This stream carries only this subscription's event notifications; it is not a general server-to-client channel." + + # === Webhook: subscribing === + - check: sep-9999-subscribe-webhook-only + text: '`events/subscribe` is ONLY used for webhook delivery. Poll and push do not need it.' + - check: sep-9999-subscribe-secret-required + text: '`delivery.secret` is REQUIRED. The client supplies the HMAC signing secret; the server never generates one.' + - check: sep-9999-subscribe-secret-format + text: 'The value MUST be a Standard Webhooks symmetric secret: the literal prefix `whsec_` followed by base64 of 24–64 random bytes.' + - check: sep-9999-subscribe-secret-rejected + text: 'Servers MUST reject a `delivery.secret` that is not `whsec_` followed by base64 decoding to 24–64 bytes (per Standard Webhooks) with `InvalidParams`.' + - check: sep-9999-subscribe-url-https-required + text: 'Callback URLs MUST use `https://`.' + - check: sep-9999-subscribe-url-non-https-rejected + text: 'Servers MUST reject `events/subscribe` with a non-`https` `delivery.url` (`-32602 InvalidParams`).' + - check: sep-9999-subscribe-auth-required + text: '`events/subscribe` and `events/unsubscribe` MUST be called with an authenticated principal; servers MUST reject calls without an authorized principal with `-32012 Forbidden`.' + - check: sep-9999-subscribe-key-composition + text: "The subscription key is `(principal, delivery.url, name, arguments)`, where `principal` is the server's canonical identifier for the authenticated subject. `arguments` is compared by canonical-JSON equality. There is no client-generated `id`." + - check: sep-9999-subscribe-key-immutable + text: "All four components are immutable for the subscription's lifetime: a subscribe call with a different value for any of them addresses a different subscription." + - check: sep-9999-subscribe-idempotent-upsert + text: '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields. If a subscription with the same scoped key exists, the server resets the TTL and updates mutable fields in place.' + - check: sep-9999-subscribe-id-derived + text: 'The server computes a deterministic `id` over the key (e.g., a truncated SHA-256 of the canonical key serialization) and returns it in the subscribe response. It is stable across refreshes and server restarts.' + - check: sep-9999-subscribe-id-not-an-input + text: "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method." + - check: sep-9999-subscribe-refresh-replaces-secret + text: 'On an idempotent subscribe against an existing key, `delivery.secret` is replaced. To rotate, the client supplies a new value on refresh.' + - check: sep-9999-subscribe-refresh-reactivates + text: 'On an idempotent subscribe against an existing key, `active` is set to `true`. If delivery had been suspended due to repeated failures, the server resumes retrying pending events.' + - check: sep-9999-subscribe-response-cursor + text: "The subscribe response carries `cursor`, a safe-to-persist watermark that advances the client's cursor even if no events arrive before next refresh." + - check: sep-9999-subscribe-response-truncated + text: 'The subscribe response carries `truncated`, true if delivery started later than the supplied cursor (retention window, `maxAgeMs` floor, or server-side ceiling).' + - check: sep-9999-subscribe-cross-tenant-isolation + text: 'Because the key includes `principal` and `delivery.url`, two distinct tenants subscribing to the same `(name, arguments)` get distinct subscriptions.' + + # === Webhook: TTL negotiation === + - check: sep-9999-ttl-refresh-before-lte-suggestion + text: '`refreshBefore` (response) is the grant. It SHOULD be less than or equal to the suggestion.' + - check: sep-9999-ttl-no-rejection-path + text: 'Clamping in either direction is self-announcing — the client reads the granted `refreshBefore` and schedules its refresh loop from that, so a clamped grant is not an error and there is no rejection path for TTL values.' + - check: sep-9999-ttl-null-only-when-requested + text: 'A server MUST NOT return `null` unless the client suggested `ttlMs: null` — no expiry exceeds every finite suggestion.' + - check: sep-9999-ttl-omitted-means-default + text: 'Omitting `ttlMs` means "server default." An explicit `ttlMs: null` requests a subscription with no expiry.' + - check: sep-9999-ttl-long-grant-retained + text: 'A server granting long or no-expiry TTLs MUST retain subscriptions for the lifetime it granted, including across restarts.' + - check: sep-9999-ttl-no-expiry-persisted + text: 'The server MUST persist no-expiry subscriptions across restarts, because a client that never refreshes will never detect (or repair) a silently dropped one.' + - check: sep-9999-ttl-no-expiry-gc-terminated + text: 'The server MAY drop a no-expiry subscription after sustained delivery failure (server-defined window), and SHOULD attempt a `terminated` envelope when it does.' + + # === Webhook: unsubscribing === + - check: sep-9999-unsubscribe-by-key + text: '`events/unsubscribe` is eager cleanup; the server looks the subscription up by the same compound key used for idempotent upsert on subscribe.' + - check: sep-9999-unsubscribe-unknown-not-found + text: 'No subscription matching the key on `events/unsubscribe` returns `-32011 NotFound`.' + + # === Webhook: delivery status === + - check: sep-9999-delivery-status-last-error-category + text: '`lastError` MUST be a server-generated category string — one of `connection_refused`, `timeout`, `tls_error`, `http_4xx`, `http_5xx`, `challenge_failed` — and MUST NOT include raw response bodies.' + + # === Webhook: delivery mechanics === + - check: sep-9999-delivery-post-json + text: 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.' + - check: sep-9999-delivery-standard-webhooks-headers + text: 'Every delivery MUST include `webhook-id` (the `eventId` for event deliveries; `msg__` for control envelopes), `webhook-timestamp` (Unix seconds), and `webhook-signature`.' + - check: sep-9999-delivery-subscription-id-header + text: 'In addition to the Standard Webhooks headers, deliveries MUST include `X-MCP-Subscription-Id` (the subscription `id`) so the receiver can select the correct secret without parsing the body. This is the only MCP-specific header.' + - check: sep-9999-delivery-signature-formula + text: 'The signature is `HMAC-SHA256(secret, webhook-id + "." + webhook-timestamp + "." + body)` encoded as base64 with a `v1,` prefix, where `body` is the raw HTTP request body bytes exactly as received and `secret` is the base64-decoded bytes of the value after the `whsec_` prefix.' + - check: sep-9999-delivery-retry-regenerates-signature + text: "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window." + - check: sep-9999-delivery-dual-sign-on-rotation + text: 'The server SHOULD dual-sign deliveries with both the old and new secrets for a short grace window so in-flight deliveries verify under either.' + - check: sep-9999-delivery-body-size + text: 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with Payload Minimality.' + - check: sep-9999-delivery-413-non-retryable + text: 'Receivers and intermediaries MAY reject larger bodies with `413 Payload Too Large`; servers MUST treat `413` as a non-retryable failure for that event.' + - check: sep-9999-delivery-410-non-retryable + text: 'A receiver that intentionally rejects a delivery and does not want it retried responds `410 Gone`; the server MUST treat it as non-retryable.' + - check: sep-9999-delivery-retries-bounded + text: 'Retries are bounded: servers SHOULD cap both the attempt count and the elapsed retry window (for example, 3–5 attempts spread over no more than 10–15 minutes).' + + # === Webhook: SSRF and endpoint verification === + - check: sep-9999-ssrf-validate-callback-url + text: 'The server MUST validate callback URLs.' + - check: sep-9999-ssrf-reject-non-routable + text: 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA IPv4 and IPv6 Special-Purpose Address Registries unless explicitly configured to allow them.' + - check: sep-9999-ssrf-validate-at-delivery-time + text: 'To prevent DNS rebinding, this validation MUST be performed at delivery time, not only at subscribe time: the server resolves the hostname, checks the resolved IP against the blocklist, and connects directly to that validated IP.' + - check: sep-9999-ssrf-no-redirects + text: 'Webhook delivery requests MUST NOT follow HTTP redirects, since a redirect can target an internal address that bypasses the blocklist.' + - check: sep-9999-verification-required-before-delivery + text: "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed, by one of: a verification handshake, a server-configured allowlist, prior out-of-band verification, or a receiver-published well-known document." + - check: sep-9999-verification-challenge-echo + text: 'Before activating, the server POSTs a `verification` control envelope carrying a single-use, short-lived `challenge` nonce (signed and headed like any delivery), and the endpoint proves intent by echoing the nonce in a `2xx` body (`{"challenge":""}`), which the server compares in constant time.' + - check: sep-9999-verification-failure-error + text: 'A reachable endpoint that fails to echo yields `-32015 CallbackEndpointError` with `data.reason: "challenge_failed"`; an unreachable one yields the same code with the relevant connection-failure category.' + - check: sep-9999-verification-cached-per-principal-url + text: "Verification is cached per `(principal, url)`: a successful handshake, allowlist hit, or well-known match covers that principal's subscriptions to that URL across refreshes and `arguments`, so varying `arguments` cannot multiply verification POSTs at a victim and one principal's verification never waives the challenge for another." + - check: sep-9999-verification-persisted-for-no-expiry + text: 'A server that persists no-expiry subscriptions across restarts MUST persist their verification status alongside them.' + - check: sep-9999-verification-uses-ssrf-hardened-path + text: 'The verification POST MUST use the same SSRF-hardened path as deliveries (delivery-time IP validation, no redirects).' + - check: sep-9999-verification-no-raw-endpoint-responses + text: 'Failures surface only via the `lastError` category `challenge_failed`, never raw endpoint responses.' + - check: sep-9999-server-identity-key-discovery + text: 'The verifying public key MUST be discovered from an origin the client already authenticates, and never from the challenge or delivery body, which the attacker controls.' + + # === Webhook: control envelopes === + - check: sep-9999-envelope-type-discriminator + text: 'A body with a top-level `type` field is a control envelope; a body without one is an `EventOccurrence`.' + - check: sep-9999-envelope-signed-like-deliveries + text: 'Control envelopes are signed and headed exactly like event deliveries (Standard Webhooks headers + `X-MCP-Subscription-Id`); the body carries a `type` discriminator instead of `eventId`/`data`.' + - check: sep-9999-envelope-webhook-id-format + text: '`webhook-id` for control envelopes is a per-message identifier of the form `msg__` so receivers can dedup retries.' + - check: sep-9999-envelope-gap + text: 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes. The client persists `cursor` and treats it as `truncated: true`.' + - check: sep-9999-envelope-terminated + text: 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended (e.g., authorization revoked). The subscription no longer exists server-side.' + + # === Authorization === + - check: sep-9999-authz-subscribe-time + text: 'When the caller is authenticated, the server MUST verify the principal has permission to subscribe to the requested event type with the given arguments.' + - check: sep-9999-authz-delivery-time-reverify + text: 'The server SHOULD periodically re-verify permissions.' + + # === Payload handling (server side) === + - check: sep-9999-payload-minimality + text: 'Servers SHOULD keep event payloads minimal — enough to identify and triage the event, not the full content.' + + # ========================================================================== + # Excluded rows. Each carries an RFC 2119 keyword in the source but is not a + # server wire obligation, so no server-side scenario can emit a check for it. + # Kept here so the keyword sweep stays auditable against the document. + # ========================================================================== + + # --- Client / host obligations --- + - text: 'The client SHOULD re-call `events/list` to refresh its event type registry.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'The client SHOULD poll again immediately (ignoring `nextPollMs`) to drain the backlog.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Clients SHOULD apply a configurable floor (default 1000 ms) to guard against a misbehaving server inducing a tight loop.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Clients SHOULD NOT apply their default request timeout to `events/stream`; a client that has received neither an event nor a heartbeat for more than twice the heartbeat interval SHOULD treat the stream as dead and reconnect with its cursor.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Unless granted no expiry, the client MUST re-call `events/subscribe` with the same subscription key before `refreshBefore` to keep the subscription alive.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'Even with no expiry, clients SHOULD still re-call `events/subscribe` (and `events/list`) occasionally.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: 'A client receiving `cursor: null` MUST NOT attempt to persist or replay from it; on reconnect/resubscribe it sends `cursor: null` (start from now).' + excluded: client-side obligation; needs a client-conformance scenario set + - text: '`truncated: true` always implies a possible gap; clients SHOULD treat it as such and persist the fresh cursor.' + excluded: client-side obligation; needs a client-conformance scenario set + - text: "Client SDKs SHOULD generate the secret on the application's behalf rather than expose an interface that encourages hand-picked values, from a CSPRNG by default." + excluded: client-SDK guidance; not observable from the server under test + - text: 'Event payloads MUST be treated with the same caution as tool results. Clients SHOULD sanitize or sandbox event payloads before presenting them to an LLM.' + excluded: host obligation; not observable from the server under test + - text: 'Clients SHOULD support policy evaluation between event receipt and action execution, and SHOULD respect server-declared governance metadata.' + excluded: host obligation (Enterprise Governance); not observable from the server under test + - text: 'The client SDK SHOULD remove the subscription and notify the application when a termination signal arrives.' + excluded: client-SDK guidance; not observable from the server under test + + # --- Receiver (callback endpoint) obligations --- + - text: 'The receiver MUST verify the signature before processing, SHOULD reject deliveries where `webhook-timestamp` is more than 5 minutes old, and SHOULD deduplicate on `webhook-id`.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'Receivers MUST compute the HMAC over the raw body, never over a re-serialized JSON object.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'The endpoint MUST make `cursor` and `eventId` available to the consuming client, and MUST forward control envelopes by the same channel it forwards events.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'The endpoint SHOULD NOT return `2xx` until the event has been durably persisted or forwarded, and SHOULD respond quickly.' + excluded: receiver obligation; the harness implements this rather than grading it + - text: 'A receiver that gets a delivery for an `id` it has not yet been told to route SHOULD return a retryable status (`503` or `425 Too Early`).' + excluded: receiver obligation; the harness implements this rather than grading it + + # --- Server-SDK implementation guidance (not wire-observable) --- + - text: "The SDK SHOULD enable poll by default so it's available unless the author opts out." + excluded: server-SDK guidance; indistinguishable on the wire from an author opting in + - text: 'SDKs SHOULD refuse a no-expiry cap unless the author has wired up durable storage.' + excluded: server-SDK guidance; a construction-time policy with no wire signal + - text: 'SDKs SHOULD provide an in-memory ring buffer that retains a bounded window of emitted events per event type.' + excluded: server-SDK guidance; an internal structure with no wire signal + - text: 'Servers SHOULD enforce the subscription limit (`ResourceExhausted`) before invoking `on_subscribe`, so a rejected subscription never provisions upstream resources.' + excluded: server-SDK ordering guidance; the harness cannot observe hook invocation order + - text: "The lease window is SDK-configurable and SHOULD default to a small multiple of the server's typical `nextPollMs`. Server authors SHOULD write `on_subscribe` to be idempotent." + excluded: server-SDK guidance; an internal lease table with no wire signal + - text: 'Authors SHOULD prefer a `check()` function that queries the upstream over emit-only for upstreams offering a durable cursor.' + excluded: server-authoring guidance; a design recommendation with no wire signal + - text: 'On HTTP/1.1 each stream is a TCP connection, so SDKs SHOULD prefer HTTP/2 when many subscriptions are active.' + excluded: transport guidance; not a protocol conformance obligation + + # --- Deployment profile advisories --- + - text: 'A WAF MAY drop requests missing any required header as a cheap pre-filter; WAF rules SHOULD NOT rely on UA matching as a security control. Servers SHOULD document their egress ranges out-of-band.' + excluded: deployment advisory addressed to operators, not to an implementation + + # --- Pure MAY / OPTIONAL (handoff disposition: no row) --- + - text: 'A server MAY return `cursor: null` in any delivery (poll result, push `notifications/events/event` and `notifications/events/active`, webhook payload) when the event type does not support replay.' + excluded: pure MAY; kept as context for sep-9999-cursor-consistency, which carries the testable half + - text: 'The one sanctioned exception is a server-side floor: a server MAY clamp an impractically short suggestion up to its minimum TTL to protect itself from refresh storms.' + excluded: pure MAY; the testable half is sep-9999-ttl-refresh-before-lte-suggestion, which this sentence excepts + - text: 'The header MAY contain multiple space-delimited signatures (`v1, v1,`) during secret rotation; the receiver accepts if any verifies.' + excluded: pure MAY, and the obligation it creates falls on the receiver + - text: 'After repeated failures (server-defined threshold), the server MAY suspend delivery (`deliveryStatus.active: false`).' + excluded: pure MAY; suspension is observable only through the OPTIONAL deliveryStatus object + - text: '`deliveryStatus` is OPTIONAL — servers MAY omit it entirely. The `events/subscribe` response MAY include a `deliveryStatus` object when refreshing an existing subscription.' + excluded: OPTIONAL by its own wording; a server omitting deliveryStatus entirely is conformant diff --git a/src/types.ts b/src/types.ts index ebe75a27..970f97e4 100644 --- a/src/types.ts +++ b/src/types.ts @@ -103,7 +103,13 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/auth/dpop', 'io.modelcontextprotocol/auth/wif', 'io.modelcontextprotocol/tasks', - 'io.modelcontextprotocol/skills' + 'io.modelcontextprotocol/skills', + // MCP Events declares its capability top-level as `capabilities.events`, + // not inside `capabilities.extensions`. This id is a suite-selection key + // only — it keeps the Events scenarios off the `--spec-version` timeline + // (see `matchesSpecVersion`), and is never a path into the capability + // object. See src/scenarios/server/events/helpers.ts. + 'io.modelcontextprotocol/events' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number]; From 1207bf85641556f3c00a44dd167c33de99b62fcb Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 17 Sep 2026 21:52:28 +0000 Subject: [PATCH 02/27] fix(events): score a server whose schemas require arguments, and grade occurrences off a poll that can carry them MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Running the suite against a second implementation — Peter Alexander's metronome-mcp.fly.dev, written by the author of the design sketch — found two bugs that kitchen-sink alone could not surface. minimalArguments gave up on any inputSchema with a required property. The document's own examples are filters and transforms with nothing required, and the helper took that for a rule. metronome.tick requires periodSeconds and label, so every poll row reported untestable and the scenario scored 1/34. It now derives a value per required property from the schema itself — default, const, first enum or examples entry, then minimum for numbers or a fixed string — and still declines to guess when a required property offers nothing. The sep-9999-occurrence-* rows were graded off the bootstrap poll, which passes cursor: null. Null means start from now, so a conformant server returns no events there: the suite could only score those seven rows against a server that violated sep-9999-cursor-null-starts-from-now. They now poll forward from the returned cursor, waiting nextPollMs between attempts (clamped to 250ms-2s) up to a 6s ceiling, and still report untestable against a quiet server. Scores with the fixes. kitchen-sink at d2950655: discovery 8/11, poll 26/29, up from 18/28 with no change to mcpkit. metronome: discovery 10/11, poll 27/29. Two metronome divergences, in the yaml header. It answers -32603 rather than -32602 for arguments that violate inputSchema. And it declares events under capabilities.extensions["io.modelcontextprotocol/events"] where this document puts it top-level, which is a question for the WG before the row moves either way. Co-Authored-By: Claude Opus 5 (1M context) --- src/scenarios/server/events/helpers.ts | 47 ++++++++++--- src/scenarios/server/events/poll.ts | 37 +++++++++- src/seps/sep-9999.yaml | 95 ++++++++++++++++++++++++-- 3 files changed, 162 insertions(+), 17 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 22cfa7c6..6b39ca38 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -312,21 +312,50 @@ export function descriptorLabel( /** * Arguments that satisfy a descriptor's `inputSchema` well enough to poll with. * - * Deliberately minimal: an empty object. Every `inputSchema` in the document is - * an object schema whose properties are filters and transforms, none of them - * required, so `{}` means "no filtering" and is valid against all of them. A - * schema that does declare `required` is the one case this cannot satisfy, and - * the caller reports that as an unmet prerequisite rather than guessing values - * a server would then reject for the wrong reason. + * Optional properties are left out, so `{}` means "no filtering". Required + * properties get a value derived from the schema itself: `default`, `const`, + * the first `enum` or `examples` entry, then a type-driven value (`minimum` + * for numbers, a fixed label for strings). A required property the schema + * gives nothing to go on for returns `undefined`, and the caller reports that + * as an unmet prerequisite rather than guessing values a server would then + * reject for the wrong reason. */ export function minimalArguments( descriptor: EventDescriptor ): Record | undefined { const schema = descriptor.inputSchema; if (!isObject(schema)) return {}; - const required = schema.required; - if (Array.isArray(required) && required.length > 0) return undefined; - return {}; + const required = Array.isArray(schema.required) ? schema.required : []; + const properties = isObject(schema.properties) ? schema.properties : {}; + const args: Record = {}; + for (const key of required) { + if (typeof key !== 'string') return undefined; + const value = schemaValue(properties[key]); + if (value === undefined) return undefined; + args[key] = value; + } + return args; +} + +function schemaValue(prop: unknown): unknown { + if (!isObject(prop)) return undefined; + if ('default' in prop) return prop.default; + if ('const' in prop) return prop.const; + if (Array.isArray(prop.enum) && prop.enum.length > 0) return prop.enum[0]; + if (Array.isArray(prop.examples) && prop.examples.length > 0) { + return prop.examples[0]; + } + switch (prop.type) { + case 'integer': + case 'number': + return typeof prop.minimum === 'number' ? prop.minimum : 1; + case 'string': + return 'mcp-conformance'; + case 'boolean': + return false; + default: + return undefined; + } } /** diff --git a/src/scenarios/server/events/poll.ts b/src/scenarios/server/events/poll.ts index 74a8845a..17d8e3e5 100644 --- a/src/scenarios/server/events/poll.ts +++ b/src/scenarios/server/events/poll.ts @@ -70,6 +70,9 @@ const POLL_IDS = [ 'sep-9999-removal-poll-not-found' ] as const; +/** How long to follow the bootstrap cursor waiting for a delivered event. */ +const OCCURRENCE_WAIT_MS = 6000; + const OCCURRENCE_IDS = [ 'sep-9999-occurrence-event-id', 'sep-9999-occurrence-name', @@ -245,7 +248,10 @@ export class EventsPollScenario implements ClientScenario { checks.push(...(await this.replayChecks(conn, name, args, r1))); // --- EventOccurrence shape ------------------------------------------- - checks.push(...this.occurrenceChecks(r1, target)); + // Not graded off r1: `cursor: null` starts from now, so a conformant + // bootstrap poll is empty. Follow its cursor until something is delivered. + const delivered = await this.pollForOccurrences(conn, name, args, r1); + checks.push(...this.occurrenceChecks(delivered, target)); // --- Error contract --------------------------------------------------- checks.push(...(await this.errorChecks(conn, descriptors, name, args))); @@ -817,6 +823,35 @@ export class EventsPollScenario implements ClientScenario { * fixture reports the whole group as untestable rather than passing an empty * array through seven shape checks, which would read as green. */ + /** + * Poll forward from the bootstrap cursor, waiting `nextPollMs` between + * attempts (clamped to 250ms-2s), until a batch carries events or + * OCCURRENCE_WAIT_MS elapses. Returns the last result either way, so a quiet + * server still reports the occurrence checks as untestable. + */ + private async pollForOccurrences( + conn: Connection, + name: string, + args: Record, + bootstrap: EventsPollResult + ): Promise { + let last = bootstrap; + const deadline = Date.now() + OCCURRENCE_WAIT_MS; + while (occurrences(last).length === 0 && Date.now() < deadline) { + const hint = typeof last.nextPollMs === 'number' ? last.nextPollMs : 1000; + const wait = Math.min(Math.max(hint, 250), 2000, deadline - Date.now()); + await new Promise((resolve) => setTimeout(resolve, wait)); + const next = await eventsPoll(conn, { + name, + arguments: args, + cursor: last.cursor ?? null + }); + if ('error' in next) return last; + last = next.result; + } + return last; + } + private occurrenceChecks( r: EventsPollResult, descriptor: EventDescriptor diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 15497836..9d61f9c1 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -127,10 +127,15 @@ # which localhost cannot satisfy: the SSRF rules declared below require a # conformant server to refuse it. # -# measured against mcpkit, 2026-09-15, examples/events/kitchen-sink at main: -# events-discovery 8/11, events-poll 18/28. Six divergences, each a real gap -# rather than something the suite should accommodate. Do not soften these -# rows to make mcpkit pass; catching them is the point. +# measured against mcpkit, 2026-09-17, examples/events/kitchen-sink at +# `d2950655`: events-discovery 8/11, events-poll 26/29. The poll number was +# 18/28 on 2026-09-15; the seven `sep-9999-occurrence-*` rows now pass +# because the scenario polls forward from the bootstrap cursor rather than +# grading them off the bootstrap response, and `sep-9999-poll-invalid- +# arguments` moved from a failure to untestable. Nothing in mcpkit changed +# between the two runs. Each divergence below is a real gap rather than +# something the suite should accommodate. Do not soften these rows to make +# mcpkit pass; catching them is the point. # # sep-9999-capability-events-object — the server answers `events/list` but # declares no `capabilities.events`, so a client that reads capabilities @@ -145,9 +150,12 @@ # empty `delivery`, where the document requires a non-empty subset of # poll/push/webhook. That source is the substitute for the missing # `notifications/events/list_changed`. Gap G34. -# sep-9999-descriptor-input-schema — no descriptor carries `inputSchema`, -# which also makes `sep-9999-poll-invalid-arguments` untestable, since -# there is no declared constraint left to violate. Not previously tracked. +# sep-9999-descriptor-input-schema — `events.topology` carries no +# `inputSchema`. mcpkit 1381 added one to the three real sources, so this +# and the delivery row above now have the same single cause. The poll +# scenario picks `alert.fired`, which has an `inputSchema` with no typed +# property, so `sep-9999-poll-invalid-arguments` reports untestable: there +# is no constraint the harness can violate on purpose. # sep-9999-poll-events-array — `events` is omitted rather than returned as # an empty array when nothing happened. Not previously tracked. # sep-9999-poll-mode-unsupported — `events/poll` answers for an event type @@ -159,6 +167,79 @@ # `UnsupportedData` (errors.go:90) has no `reason` field, but driving that # path needs a server that can drop an event type mid-run, which waits on # the push scenario. +# +# measured against a second implementation, 2026-09-17: Peter Alexander's +# `https://metronome-mcp.fly.dev/mcp`, announced 2026-09-10 and written by +# the author of the design sketch. events-discovery 10/11, events-poll 27/29. +# It serves one event type, `metronome.tick`, which offers all three delivery +# modes. +# +# Running it is not a one-liner. Three things differ from a local fixture: +# +# Events are only on the 2026-07-28 stateless wire. Over 2025-11-25 the +# server answers `events/list` with -32601. The runner picks the wire from +# the spec version, and the default already lands on stateless, so no flag +# is needed — but a 2025-11-25 run reports the whole extension as absent. +# It is behind OAuth, and `server` has no way to send a token. Its +# authorization server is a demo that verifies no identity: register a +# client at /register, approve the consent form, exchange the code with +# PKCE. Then put a local proxy in front that adds the header and point +# --url at the proxy. An auth option on the runner would remove all of +# this, and the webhook scenarios will need one anyway, since a webhook +# subscription requires an authenticated principal. +# Its one event type has required `inputSchema` properties. See the +# `minimalArguments` note below. +# +# Two divergences, neither previously known: +# +# sep-9999-capability-events-object — metronome declares events under +# `capabilities.extensions["io.modelcontextprotocol/events"]`, the SEP-2133 +# extensions map, where this document puts it top-level under +# `capabilities`. Ask before changing the row. The document says top-level +# and has not moved since `28ec35e9`, but metronome is the sketch author's +# own server, and every other extension in MCP declares itself through the +# extensions map. If the WG confirms the extensions map, this row and +# `EVENTS_CAPABILITY` in src/scenarios/server/events/helpers.ts both change, +# and mcpkit's failure on the same row means something different than it +# does today. +# sep-9999-poll-invalid-arguments — arguments that violate `inputSchema` +# answer -32603 Internal error, where the document requires -32602 +# InvalidParams. The message ("Got JSON of type string, expected int_") +# reads like a deserialization failure surfacing rather than a validation +# step, so it is likely a framework default rather than a decision. +# +# `sep-9999-poll-mode-unsupported` reports untestable here, for the opposite +# reason it fails against mcpkit: every event type metronome serves offers +# poll, so there is no type to probe the unsupported-mode path with. The two +# implementations cover each other's blind spot, which is the argument for +# keeping both targets rather than picking one. +# +# two suite bugs the second implementation found, both fixed here: +# +# `minimalArguments` (helpers.ts) returned undefined for any `inputSchema` +# with a required property, on the reasoning that every schema in the +# document is filters and transforms with nothing required. That holds for +# the examples in the document, not for a real server: `metronome.tick` +# requires `periodSeconds` and `label`, and all 33 poll rows reported +# untestable, scoring 1/34. It now derives a value per required property +# from the schema — `default`, `const`, first `enum` or `examples` entry, +# then `minimum` for numbers or a fixed string — and still gives up, rather +# than guessing, when a required property offers nothing to go on. +# The `sep-9999-occurrence-*` rows were graded off the bootstrap poll, which +# passes `cursor: null`. `cursor: null` means start from now, so a +# conformant server returns no events and the seven shape rows could never +# pass: the suite required the server to violate +# `sep-9999-cursor-null-starts-from-now` to score them. They now poll +# forward from the returned cursor, waiting `nextPollMs` between attempts +# (clamped to 250ms-2s) up to a 6s ceiling, and still report untestable +# against a genuinely quiet server. This is why the mcpkit poll number +# moved without mcpkit changing. +# +# Both bugs were invisible against kitchen-sink alone. The first needs a +# server whose schema requires something; the second needs one that does not +# replay on a null cursor. This is the same pattern as the SEP-2640 suite, +# where the two bugs that mattered came from running against something that +# was not ours. sep: 9999 spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md requirements: From 14457c12e6c2802a505a1cdddc5170e0c7e0cd62 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sun, 20 Sep 2026 19:28:28 +0000 Subject: [PATCH 03/27] feat(events): events-push, server conformance for push delivery Adds the third of five events scenarios, covering the 17 sep-9999-stream-* rows. The Connection abstraction cannot express events/stream: conn.request resolves on the response for its id, and a push stream withholds that until the subscription ends, so a scenario awaiting it would block for the life of the stream and see none of the notifications it grades. stream.ts opens the POST itself and hands back a session while the stream is still open. Aborting that request is also the document's client-side cancel on Streamable HTTP, so the cancel row falls out of the same mechanism. The watch window is 35s by default, over the runner's 30s timeout, so the scenario needs --timeout 60000. That is deliberate: the heartbeat is a MUST with a 30s SHOULD on cadence, so a shorter window cannot tell a silent server from a slow one. Under 30s the heartbeat rows report untestable rather than failing a server that may be conformant. The interval check carries 2s of tolerance, because kitchen-sink times its heartbeat at exactly 30s and measured 30005ms once scheduling had its say. Rows needing something the client cannot ask for - an upstream failure, a retention gap, a termination, a server-initiated close - report untestable with the prerequisite named, per src/scenarios/untestable.ts. kitchen-sink at d2950655: 11/15. metronome: 13/15, where both failures are untestable rows rather than divergences. One new divergence, not previously tracked: kitchen-sink puts the correlation id in params.requestId, where the document requires params._meta["io.modelcontextprotocol/subscriptionId"]. A client holding two streams cannot route by what the document tells it to read. --- src/scenarios/index.ts | 2 + src/scenarios/server/events/push.ts | 796 ++++++++++++++++++++++++++ src/scenarios/server/events/stream.ts | 238 ++++++++ 3 files changed, 1036 insertions(+) create mode 100644 src/scenarios/server/events/push.ts create mode 100644 src/scenarios/server/events/stream.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index cf43bda3..58e3a1b7 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -62,6 +62,7 @@ import { import { EventsDiscoveryScenario } from './server/events/discovery'; import { EventsPollScenario } from './server/events/poll'; +import { EventsPushScenario } from './server/events/push'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; @@ -238,6 +239,7 @@ const allClientScenariosList: ClientScenario[] = [ // each scenario SKIPs cleanly when the capability is not declared. new EventsDiscoveryScenario(), new EventsPollScenario(), + new EventsPushScenario(), // Prompts scenarios new PromptsListScenario(), diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts new file mode 100644 index 00000000..222e5418 --- /dev/null +++ b/src/scenarios/server/events/push.ts @@ -0,0 +1,796 @@ +/** + * MCP Events: push delivery over a long-lived `events/stream` request. + * + * Scored against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events. Each check's verbatim + * excerpt lives beside its id in src/seps/sep-9999.yaml, where 9999 is a + * placeholder SEP number. + * + * Two things shape what this scenario can grade. + * + * The heartbeat is a MUST with a SHOULD of "at least every 30 seconds", so a + * conformant server may say nothing for 30s. The observation window therefore + * has to outlast that, which is why the default is 35s and why the scenario + * needs `--timeout 60000` rather than the runner's 30s default. A shorter + * window would report a compliant slow-heartbeat server as broken, which is + * worse than taking the time. + * + * Several rows need a server doing something the harness cannot ask for: an + * upstream failure (`stream-error-is-recoverable`), a retention gap + * (`stream-gap-resends-active`), a termination (`stream-terminated-*`), or a + * server-initiated close (`stream-final-result-*`). Those report untestable + * with the missing prerequisite named, per the untestable policy in + * src/scenarios/untestable.ts, rather than passing vacuously against a server + * that simply never did it. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { RunContext } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_ACTIVE_NOTIFICATION, + EVENTS_CAPABILITY, + EVENTS_ERROR_NOTIFICATION, + EVENTS_EVENT_NOTIFICATION, + EVENTS_EXTENSION_ID, + EVENTS_HEARTBEAT_NOTIFICATION, + EVENTS_SPEC_REF, + EVENTS_STREAM_METHOD, + EVENTS_TERMINATED_NOTIFICATION, + EVENTS_NOT_FOUND, + JSONRPC_METHOD_NOT_FOUND, + SUBSCRIPTION_ID_META, + describeValue, + descriptorName, + eventsCheck, + eventsListAll, + firstSupporting, + isIso8601, + isObject, + isValidCursor, + minimalArguments +} from './helpers'; +import { openEventStream, type StreamSession } from './stream'; + +/** How long to watch an idle stream. Must outlast the document's 30s SHOULD. */ +const WATCH_MS = Number(process.env.EVENTS_PUSH_WATCH_MS ?? 35000); + +/** How long to wait for the subscription confirmation before grading it. */ +const ACTIVE_MS = 3000; + +/** Slack on the 30s heartbeat SHOULD, for scheduling and network jitter. */ +const HEARTBEAT_TOLERANCE_MS = 2000; + +const STREAM_IDS = [ + 'sep-9999-stream-implemented', + 'sep-9999-stream-error-before-open', + 'sep-9999-stream-active-confirmation', + 'sep-9999-stream-subscription-id-meta', + 'sep-9999-stream-event-notification', + 'sep-9999-stream-error-is-recoverable', + 'sep-9999-stream-terminated-ends-subscription', + 'sep-9999-stream-gap-resends-active', + 'sep-9999-stream-heartbeat-required', + 'sep-9999-stream-heartbeat-carries-cursor', + 'sep-9999-stream-heartbeat-interval', + 'sep-9999-stream-heartbeat-not-sse-comment', + 'sep-9999-stream-final-result-shape', + 'sep-9999-stream-final-result-timing', + 'sep-9999-stream-cancel-stops-delivery', + 'sep-9999-stream-exempt-from-concurrency-cap', + 'sep-9999-stream-carries-only-event-notifications' +] as const; + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +function skipAll(reason: string): ConformanceCheck[] { + return STREAM_IDS.map((id) => + eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsPushScenario implements ClientScenario { + name = 'events-push'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: push delivery over a long-lived \`events/stream\` request. + +**Methods**: \`events/stream\`, plus \`events/list\` to discover an event type advertising \`push\` delivery + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-stream-implemented\` / \`sep-9999-stream-error-before-open\` — a valid subscription opens a stream; an invalid one answers a JSON-RPC error and opens nothing +- \`sep-9999-stream-active-confirmation\` / \`sep-9999-stream-subscription-id-meta\` — the \`notifications/events/active\` confirmation, and the parent request id echoed on every notification +- \`sep-9999-stream-event-notification\` / \`sep-9999-stream-carries-only-event-notifications\` — events arrive as \`notifications/events/event\`, and nothing else rides the stream +- \`sep-9999-stream-heartbeat-*\` — the keepalive is required, carries a cursor, arrives at least every 30s, and is a \`data:\` frame rather than an SSE comment +- \`sep-9999-stream-exempt-from-concurrency-cap\` — concurrent streams stay open together +- \`sep-9999-stream-cancel-stops-delivery\` / \`sep-9999-stream-final-result-*\` — cancellation and the empty final result + +**This scenario needs \`--timeout 60000\`.** It watches an idle stream for ${WATCH_MS / 1000}s, because a server may heartbeat as slowly as every 30s and still be conformant. Override the window with \`EVENTS_PUSH_WATCH_MS\`. + +**Untestable rather than green**: an upstream failure, a retention gap, a termination and a server-initiated close cannot be provoked from the client side, so those rows name the missing prerequisite instead of passing against a server that simply never did it.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + let declared = false; + try { + const capabilities = await conn.discover(); + const caps = isObject(capabilities.capabilities) + ? capabilities.capabilities + : {}; + declared = caps[EVENTS_CAPABILITY] !== undefined; + + const listed = await eventsListAll(conn); + if ('error' in listed) { + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + STREAM_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no push-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const target = firstSupporting(listed.descriptors, 'push'); + const name = target ? descriptorName(target) : undefined; + if (!target || !name) { + return untestableAll( + STREAM_IDS, + listed.descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no push-capable event type could be exercised.' + : 'No event type advertises `push` delivery, so `events/stream` could not be exercised. Push is optional per event type.' + ); + } + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + STREAM_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema, so no stream could be opened.` + ); + } + + return await this.streamChecks(ctx, name, args); + } finally { + await conn.close(); + } + } + + private async streamChecks( + ctx: RunContext, + name: string, + args: Record + ): Promise { + const checks: ConformanceCheck[] = []; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + + try { + // --- The stream opened at all --------------------------------------- + if (!session.contentType?.includes('text/event-stream')) { + const err = session.error; + checks.push( + eventsCheck( + 'sep-9999-stream-implemented', + 'Push delivery uses a long-lived `events/stream` request — one per subscription, a standard JSON-RPC request with an `id`.', + 'FAILURE', + { + errorMessage: err + ? `Event type \`${name}\` advertises \`push\` delivery but \`${EVENTS_STREAM_METHOD}\` answered ${err.code} ${err.message} instead of opening a stream.` + : `\`${EVENTS_STREAM_METHOD}\` answered HTTP ${session.status} with content-type ${describeValue(session.contentType)}, expected an SSE stream.`, + details: { status: session.status, error: err } + } + ) + ); + checks.push( + ...untestableAll( + STREAM_IDS.filter((id) => id !== 'sep-9999-stream-implemented'), + `No stream was opened for \`${name}\`, so nothing on it could be observed.` + ) + ); + checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); + return dedupe(checks); + } + + checks.push( + eventsCheck( + 'sep-9999-stream-implemented', + 'Push delivery uses a long-lived `events/stream` request — one per subscription, a standard JSON-RPC request with an `id`.', + 'SUCCESS', + { details: { name, status: session.status } } + ) + ); + + // --- Confirmation ---------------------------------------------------- + const active = await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + checks.push(this.activeCheck(active, session)); + + // --- Watch the stream ------------------------------------------------ + // One window serves every timing row: heartbeats, delivered events, and + // whatever else the server chooses to put on the stream. + await session.settle(WATCH_MS); + + checks.push(...this.notificationChecks(session, name)); + checks.push(...this.heartbeatChecks(session)); + + // --- Cancellation ------------------------------------------------------ + const beforeCancel = session.notifications.length; + await session.cancel(); + await sleep(500); + checks.push( + session.notifications.length === beforeCancel + ? eventsCheck( + 'sep-9999-stream-cancel-stops-delivery', + 'On cancellation the server MUST stop delivering events and release any associated resources.', + 'SUCCESS', + { + details: { + note: 'Aborting the request stream is the Streamable HTTP cancel; no further frames were read.', + notifications: beforeCancel + } + } + ) + : eventsCheck( + 'sep-9999-stream-cancel-stops-delivery', + 'On cancellation the server MUST stop delivering events and release any associated resources.', + 'FAILURE', + { + errorMessage: `${session.notifications.length - beforeCancel} further notification(s) arrived after the request stream was aborted.` + } + ) + ); + + checks.push(...this.finalResultChecks(session)); + checks.push(...(await this.concurrencyChecks(ctx, name, args))); + checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); + return dedupe(checks); + } finally { + await session.cancel(); + } + } + + /** `notifications/events/active {cursor, truncated, _meta.subscriptionId}`. */ + private activeCheck( + active: { params: Record } | undefined, + session: StreamSession + ): ConformanceCheck { + const description = + 'The server confirms the subscription with `notifications/events/active {cursor, truncated, _meta.subscriptionId}`.'; + if (!active) { + return eventsCheck( + 'sep-9999-stream-active-confirmation', + description, + 'FAILURE', + { + errorMessage: `No \`${EVENTS_ACTIVE_NOTIFICATION}\` arrived within ${ACTIVE_MS}ms of the stream opening.`, + details: { + notifications: session.notifications.map((n) => n.method) + } + } + ); + } + const problems: string[] = []; + if (!isValidCursor(active.params.cursor)) { + problems.push( + `\`cursor\` is ${describeValue(active.params.cursor)}, expected a string or null` + ); + } + if ( + active.params.truncated !== undefined && + typeof active.params.truncated !== 'boolean' + ) { + problems.push( + `\`truncated\` is ${describeValue(active.params.truncated)}, expected a boolean` + ); + } + return problems.length === 0 + ? eventsCheck( + 'sep-9999-stream-active-confirmation', + description, + 'SUCCESS', + { details: { params: active.params } } + ) + : eventsCheck( + 'sep-9999-stream-active-confirmation', + description, + 'FAILURE', + { + errorMessage: problems.join('; '), + details: { params: active.params } + } + ); + } + + /** What rode the stream, and whether every frame was routable. */ + private notificationChecks( + session: StreamSession, + name: string + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const notifications = session.notifications; + + // Every notifications/events/* message carries the parent request id. + const missingMeta = notifications.filter((n) => { + const meta = isObject(n.params._meta) ? n.params._meta : {}; + return meta[SUBSCRIPTION_ID_META] !== session.requestId; + }); + out.push( + missingMeta.length === 0 && notifications.length > 0 + ? eventsCheck( + 'sep-9999-stream-subscription-id-meta', + 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]`.', + 'SUCCESS', + { details: { checked: notifications.length } } + ) + : notifications.length === 0 + ? untestableCheck( + 'sep-9999-stream-subscription-id-meta', + 'sep-9999-stream-subscription-id-meta', + 'Every `notifications/events/*` message carries the parent request id in `_meta`.', + 'The stream delivered no notifications, so no message could be checked for the correlation id.', + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-9999-stream-subscription-id-meta', + 'Every `notifications/events/*` message carries the JSON-RPC `id` of the parent `events/stream` request in `params._meta["io.modelcontextprotocol/subscriptionId"]`.', + 'FAILURE', + { + errorMessage: `${missingMeta.length} of ${notifications.length} notification(s) did not carry \`${SUBSCRIPTION_ID_META}\` = ${session.requestId}.`, + details: { + offending: missingMeta + .slice(0, 3) + .map((n) => ({ method: n.method, meta: n.params._meta })) + } + } + ) + ); + + // The stream carries only notifications/events/*. + const foreign = notifications.filter( + (n) => !n.method.startsWith('notifications/events/') + ); + out.push( + foreign.length === 0 + ? eventsCheck( + 'sep-9999-stream-carries-only-event-notifications', + 'The `events/stream` response carries only `notifications/events/*` messages; it is not a general server-to-client channel.', + 'SUCCESS', + { details: { checked: notifications.length } } + ) + : eventsCheck( + 'sep-9999-stream-carries-only-event-notifications', + 'The `events/stream` response carries only `notifications/events/*` messages; it is not a general server-to-client channel.', + 'FAILURE', + { + errorMessage: `Non-event messages rode the stream: ${[...new Set(foreign.map((n) => n.method))].join(', ')}.` + } + ) + ); + + // Delivered events. + const events = notifications.filter( + (n) => n.method === EVENTS_EVENT_NOTIFICATION + ); + if (events.length === 0) { + out.push( + untestableCheck( + 'sep-9999-stream-event-notification', + 'sep-9999-stream-event-notification', + 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.', + `No event was delivered for \`${name}\` in ${WATCH_MS}ms. The fixture needs an event type that emits while the stream is open.`, + [EVENTS_SPEC_REF] + ) + ); + } else { + const bad = events.filter((n) => { + const p = n.params; + return ( + typeof p.eventId !== 'string' || + typeof p.name !== 'string' || + !isIso8601(p.timestamp) || + !isObject(p.data) + ); + }); + out.push( + bad.length === 0 + ? eventsCheck( + 'sep-9999-stream-event-notification', + 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.', + 'SUCCESS', + { details: { delivered: events.length } } + ) + : eventsCheck( + 'sep-9999-stream-event-notification', + 'Events are delivered as `notifications/events/event` whose params are an `EventOccurrence`.', + 'FAILURE', + { + errorMessage: `${bad.length} of ${events.length} event notification(s) were not a valid EventOccurrence (eventId, name, timestamp and data are required).`, + details: { first: bad[0]?.params } + } + ) + ); + } + + // Rows that need the server to do something the harness cannot ask for. + const errorNotes = notifications.filter( + (n) => n.method === EVENTS_ERROR_NOTIFICATION + ); + out.push( + errorNotes.length === 0 + ? untestableCheck( + 'sep-9999-stream-error-is-recoverable', + 'sep-9999-stream-error-is-recoverable', + '`notifications/events/error` reports a recoverable failure; the subscription remains active.', + 'No upstream failure occurred during the run, and the harness cannot provoke one. Needs a fixture whose upstream can be made to fail on demand.', + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-9999-stream-error-is-recoverable', + '`notifications/events/error` reports a recoverable failure; the subscription remains active and the server retries and resumes.', + session.open ? 'SUCCESS' : 'FAILURE', + { + errorMessage: session.open + ? undefined + : 'The stream closed after `notifications/events/error`, but only `notifications/events/terminated` ends a subscription.', + details: { errors: errorNotes.length } + } + ) + ); + + const terminated = notifications.filter( + (n) => n.method === EVENTS_TERMINATED_NOTIFICATION + ); + out.push( + terminated.length === 0 + ? untestableCheck( + 'sep-9999-stream-terminated-ends-subscription', + 'sep-9999-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + 'The subscription was not terminated during the run. Needs a server that can revoke authorization or remove an event type mid-stream.', + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-9999-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + 'SUCCESS', + { details: { terminated: terminated.length } } + ) + ); + + const actives = notifications.filter( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION + ); + const gapActive = actives.slice(1).find((n) => n.params.truncated === true); + out.push( + gapActive + ? eventsCheck( + 'sep-9999-stream-gap-resends-active', + 'A gap is not an error — the server sends a fresh `notifications/events/active {cursor:, truncated:true}` and continues delivering.', + 'SUCCESS', + { details: { params: gapActive.params } } + ) + : untestableCheck( + 'sep-9999-stream-gap-resends-active', + 'sep-9999-stream-gap-resends-active', + 'A gap is signalled by a fresh `notifications/events/active` with `truncated: true`, not an error.', + 'No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture that can expire its replay window on demand.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** The four heartbeat rows, all graded off the same watch window. */ + private heartbeatChecks(session: StreamSession): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const beats = session.notifications.filter( + (n) => n.method === EVENTS_HEARTBEAT_NOTIFICATION + ); + + // A window shorter than the 30s cadence the document allows cannot tell a + // silent server from a slow one, so it reports the missing prerequisite + // rather than failing a server that may be conformant. + const windowCoversCadence = WATCH_MS > 30000; + out.push( + beats.length > 0 + ? eventsCheck( + 'sep-9999-stream-heartbeat-required', + 'The server MUST send periodic keepalive messages on the push stream so the client can distinguish "nothing to send" from "connection is dead."', + 'SUCCESS', + { details: { beats: beats.length, windowMs: WATCH_MS } } + ) + : windowCoversCadence + ? eventsCheck( + 'sep-9999-stream-heartbeat-required', + 'The server MUST send periodic keepalive messages on the push stream so the client can distinguish "nothing to send" from "connection is dead."', + 'FAILURE', + { + errorMessage: `No \`${EVENTS_HEARTBEAT_NOTIFICATION}\` arrived in ${WATCH_MS}ms, which outlasts the 30s cadence the document asks for.`, + details: { + windowMs: WATCH_MS, + sawSseComments: session.sseComments.length + } + } + ) + : untestableCheck( + 'sep-9999-stream-heartbeat-required', + 'sep-9999-stream-heartbeat-required', + 'The server MUST send periodic keepalive messages on the push stream.', + `No heartbeat arrived, but the watch window was ${WATCH_MS}ms and the document allows a 30s cadence, so a conformant server could look identical. Re-run with EVENTS_PUSH_WATCH_MS above 30000.`, + [EVENTS_SPEC_REF] + ) + ); + + if (beats.length === 0) { + out.push( + ...untestableAll( + [ + 'sep-9999-stream-heartbeat-carries-cursor', + 'sep-9999-stream-heartbeat-interval' + ], + `No heartbeat arrived in ${WATCH_MS}ms, so its contents and cadence could not be observed.` + ) + ); + } else { + const badCursor = beats.filter((n) => !isValidCursor(n.params.cursor)); + out.push( + badCursor.length === 0 + ? eventsCheck( + 'sep-9999-stream-heartbeat-carries-cursor', + 'The heartbeat carries `cursor`, the position the server has checked up to; it is `null` for event types that do not support replay.', + 'SUCCESS', + { details: { beats: beats.length } } + ) + : eventsCheck( + 'sep-9999-stream-heartbeat-carries-cursor', + 'The heartbeat carries `cursor`, the position the server has checked up to; it is `null` for event types that do not support replay.', + 'FAILURE', + { + errorMessage: `${badCursor.length} heartbeat(s) carried a \`cursor\` that was neither a string nor null (first: ${describeValue(badCursor[0]?.params.cursor)}).` + } + ) + ); + + // The gap the client would see: stream open to first beat, then between + // beats. A single beat still bounds the wait the client endured. + const marks = [0, ...beats.map((b) => b.atMs)]; + const gaps = marks.slice(1).map((m, i) => m - marks[i]); + const worst = Math.max(...gaps); + // A server that times its heartbeat at exactly 30s lands a few ms over + // once scheduling and the network have had their say, and reporting that + // as a missed SHOULD is the harness being pedantic rather than the server + // being late. kitchen-sink measured 30005ms. + out.push( + worst <= 30000 + HEARTBEAT_TOLERANCE_MS + ? eventsCheck( + 'sep-9999-stream-heartbeat-interval', + 'The server SHOULD send a heartbeat at least every 30 seconds.', + 'SUCCESS', + { details: { worstGapMs: worst, beats: beats.length } } + ) + : eventsCheck( + 'sep-9999-stream-heartbeat-interval', + 'The server SHOULD send a heartbeat at least every 30 seconds.', + 'WARNING', + { + errorMessage: `Longest silence was ${worst}ms, over the 30s the document asks for (with ${HEARTBEAT_TOLERANCE_MS}ms of tolerance).`, + details: { gapsMs: gaps } + } + ) + ); + } + + out.push( + session.sseComments.length === 0 + ? eventsCheck( + 'sep-9999-stream-heartbeat-not-sse-comment', + 'On Streamable HTTP the heartbeat is an SSE `data:` frame; the SSE comment form (`: keepalive`) is not used since it cannot carry cursor state.', + 'SUCCESS', + { details: { comments: 0 } } + ) + : eventsCheck( + 'sep-9999-stream-heartbeat-not-sse-comment', + 'On Streamable HTTP the heartbeat is an SSE `data:` frame; the SSE comment form (`: keepalive`) is not used since it cannot carry cursor state.', + beats.length === 0 ? 'FAILURE' : 'WARNING', + { + errorMessage: `The stream carried ${session.sseComments.length} SSE comment line(s) (e.g. ${JSON.stringify(session.sseComments[0])}), which cannot carry cursor state.` + } + ) + ); + + return out; + } + + /** The `StreamEventsResult`, which only a server-initiated close produces. */ + private finalResultChecks(session: StreamSession): ConformanceCheck[] { + const result = session.finalResult?.result; + if (result === undefined) { + return untestableAll( + [ + 'sep-9999-stream-final-result-shape', + 'sep-9999-stream-final-result-timing' + ], + 'The harness cancelled the stream, and on Streamable HTTP a client-side abort is terminal, so no final frame is sent. Grading this needs a server that closes the stream itself.', + 'WARNING' + ); + } + const out: ConformanceCheck[] = []; + const keys = isObject(result) + ? Object.keys(result).filter((k) => k !== '_meta') + : ['']; + out.push( + keys.length === 0 + ? eventsCheck( + 'sep-9999-stream-final-result-shape', + 'The `StreamEventsResult` is an empty typed result (`{"_meta": {}}`).', + 'SUCCESS', + { details: { result } } + ) + : eventsCheck( + 'sep-9999-stream-final-result-shape', + 'The `StreamEventsResult` is an empty typed result (`{"_meta": {}}`).', + 'FAILURE', + { + errorMessage: `The final result carried ${keys.join(', ')}; it is defined to carry no information.`, + details: { result } + } + ) + ); + out.push( + eventsCheck( + 'sep-9999-stream-final-result-timing', + 'The final result is sent whenever the server can write a final frame; on Streamable HTTP only when the server initiates the close.', + 'SUCCESS', + { + details: { + note: 'The server initiated the close and wrote a final frame.' + } + } + ) + ); + return out; + } + + /** Concurrent streams, which a request-concurrency cap would strangle. */ + private async concurrencyChecks( + ctx: RunContext, + name: string, + args: Record + ): Promise { + const description = + 'Server SDKs MUST exempt `events/stream` from any general request-concurrency cap, since each push subscription is a long-lived request that never completes until cancelled.'; + const sessions: StreamSession[] = []; + try { + for (let i = 0; i < 3; i++) { + sessions.push( + await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ) + ); + } + const streamed = sessions.filter((s) => + s.contentType?.includes('text/event-stream') + ); + const confirmed = await Promise.all( + streamed.map((s) => + s.waitFor((n) => n.method === EVENTS_ACTIVE_NOTIFICATION, ACTIVE_MS) + ) + ); + const live = confirmed.filter(Boolean).length; + return [ + live === sessions.length + ? eventsCheck( + 'sep-9999-stream-exempt-from-concurrency-cap', + description, + 'SUCCESS', + { details: { concurrent: live } } + ) + : eventsCheck( + 'sep-9999-stream-exempt-from-concurrency-cap', + description, + 'FAILURE', + { + errorMessage: `Opened ${sessions.length} concurrent \`${EVENTS_STREAM_METHOD}\` requests; only ${live} confirmed with \`${EVENTS_ACTIVE_NOTIFICATION}\`.`, + details: { + statuses: sessions.map((s) => ({ + status: s.status, + contentType: s.contentType, + error: s.error + })) + } + } + ) + ]; + } finally { + await Promise.all(sessions.map((s) => s.cancel())); + } + } + + /** An invalid subscription answers an error and opens no stream. */ + private async errorBeforeOpenChecks( + ctx: RunContext, + args: Record + ): Promise { + const unknown = `conformance.nonexistent.${Date.now()}`; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name: unknown, arguments: args, cursor: null }, + { openTimeoutMs: 1000 } + ); + try { + const description = + 'If the subscription is invalid (`NotFound`, `Forbidden`, `InvalidParams`, `Unsupported`), the server responds immediately with a JSON-RPC error and no stream is opened.'; + if (!session.error) { + return [ + eventsCheck( + 'sep-9999-stream-error-before-open', + description, + 'FAILURE', + { + errorMessage: `\`${EVENTS_STREAM_METHOD}\` for unknown event type \`${unknown}\` did not answer a JSON-RPC error (HTTP ${session.status}, content-type ${describeValue(session.contentType)}).`, + details: { + notifications: session.notifications.map((n) => n.method) + } + } + ) + ]; + } + return [ + session.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-9999-stream-error-before-open', + description, + 'SUCCESS', + { details: { code: session.error.code } } + ) + : eventsCheck( + 'sep-9999-stream-error-before-open', + description, + 'WARNING', + { + errorMessage: `Unknown event type answered ${session.error.code} ${session.error.message}; the document names \`-32011 NotFound\` for a referenced entity that does not exist.`, + details: { error: session.error } + } + ) + ]; + } finally { + await session.cancel(); + } + } +} + +/** Keep the first check emitted per id, so a fallback path cannot double-report. */ +function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { + const seen = new Set(); + return checks.filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/src/scenarios/server/events/stream.ts b/src/scenarios/server/events/stream.ts new file mode 100644 index 00000000..6b58c85b --- /dev/null +++ b/src/scenarios/server/events/stream.ts @@ -0,0 +1,238 @@ +/** + * A long-lived `events/stream` request, which the `Connection` abstraction + * cannot express. + * + * `conn.request()` resolves when the response for its JSON-RPC id arrives, and + * that is exactly what a push stream withholds: the `StreamEventsResult` is + * the last frame before close, so a scenario that awaited it would block for + * the life of the subscription and see none of the notifications it is meant + * to grade. `readSseJsonRpcResponse` has the same shape of problem — it stops + * reading at the first frame matching the request id. + * + * So this opens the POST itself, hands back a session while the stream is + * still open, and appends frames as they arrive. The scenario waits for the + * frames it needs, grades them, then aborts. Aborting is also what the + * document calls the client-side cancel on Streamable HTTP, so the cancel + * checks come out of the same mechanism rather than a second code path. + * + * stdio is out of scope here: the runner addresses a URL, and the document's + * stdio rules (`notifications/cancelled`, the MAY on a final result) have no + * HTTP analogue. Rows that only bite on stdio are reported untestable by the + * scenario rather than graded against a transport the harness cannot open. + */ + +import { + buildStandardHeaders, + withRequestMeta, + type JsonRpcResponse +} from '../../../connection'; +import type { SpecVersion } from '../../../types'; +import { isObject } from './helpers'; + +/** A JSON-RPC notification as it arrived on the stream. */ +export interface StreamNotification { + method: string; + params: Record; + /** Milliseconds since the stream was opened, for the heartbeat interval. */ + atMs: number; +} + +export interface StreamSession { + /** The JSON-RPC id of the `events/stream` request, echoed in `_meta`. */ + readonly requestId: number; + /** HTTP status of the POST that opened the stream. */ + readonly status: number; + readonly contentType?: string; + /** Set when the server answered with an immediate JSON-RPC error. */ + readonly error?: { code: number; message: string; data?: unknown }; + /** Set when a response for `requestId` arrived (the `StreamEventsResult`). */ + readonly finalResult?: JsonRpcResponse; + /** Notifications in arrival order. */ + readonly notifications: StreamNotification[]; + /** Frames that were neither a notification nor this request's response. */ + readonly foreignFrames: unknown[]; + /** SSE comment lines (`: keepalive`), which the document rules out. */ + readonly sseComments: string[]; + /** Whether the reader is still running. */ + readonly open: boolean; + /** Resolve once a notification matches, or undefined at the deadline. */ + waitFor( + predicate: (n: StreamNotification) => boolean, + timeoutMs: number + ): Promise; + /** Resolve after `ms`, or earlier if the stream closes. */ + settle(ms: number): Promise; + /** Abort the request stream, which is the client-side cancel on HTTP. */ + cancel(): Promise; +} + +/** Open `events/stream` and return once the server has answered the POST. */ +export async function openEventStream( + serverUrl: string, + specVersion: SpecVersion, + params: Record, + options: { openTimeoutMs?: number } = {} +): Promise { + const requestId = Math.floor(Math.random() * 1_000_000) + 1; + const headers = buildStandardHeaders('events/stream', params, { + specVersion + }); + const body = JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + method: 'events/stream', + params: withRequestMeta(params, specVersion) + }); + + const controller = new AbortController(); + const openedAt = Date.now(); + const notifications: StreamNotification[] = []; + const foreignFrames: unknown[] = []; + const sseComments: string[] = []; + const state = { + open: true, + error: undefined as StreamSession['error'], + finalResult: undefined as JsonRpcResponse | undefined + }; + + const res = await fetch(serverUrl, { + method: 'POST', + headers, + body, + signal: controller.signal + }); + const contentType = res.headers.get('content-type') ?? undefined; + + const ingest = (frame: unknown): void => { + if (!isObject(frame)) { + foreignFrames.push(frame); + return; + } + if (typeof frame.method === 'string' && frame.id === undefined) { + notifications.push({ + method: frame.method, + params: isObject(frame.params) ? frame.params : {}, + atMs: Date.now() - openedAt + }); + return; + } + if (frame.id === requestId) { + state.finalResult = frame as unknown as JsonRpcResponse; + if (isObject(frame.error)) { + state.error = { + code: Number(frame.error.code), + message: String(frame.error.message ?? ''), + data: frame.error.data + }; + } + return; + } + foreignFrames.push(frame); + }; + + // A JSON body means the server answered rather than streamed: either the + // immediate error the document requires for an invalid subscription, or a + // server that does not implement push at all. + if (!contentType?.includes('text/event-stream')) { + state.open = false; + const text = await res.text(); + try { + ingest(JSON.parse(text)); + } catch { + foreignFrames.push(text); + } + } else { + void readFrames(res, ingest, sseComments).finally(() => { + state.open = false; + }); + } + + const session: StreamSession = { + requestId, + status: res.status, + contentType, + get error() { + return state.error; + }, + get finalResult() { + return state.finalResult; + }, + get open() { + return state.open; + }, + notifications, + foreignFrames, + sseComments, + async waitFor(predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const hit = notifications.find(predicate); + if (hit) return hit; + if (!state.open || Date.now() >= deadline) { + return notifications.find(predicate); + } + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async settle(ms) { + const deadline = Date.now() + ms; + while (state.open && Date.now() < deadline) { + await sleep(Math.min(100, Math.max(1, deadline - Date.now()))); + } + }, + async cancel() { + controller.abort(); + state.open = false; + } + }; + + // Give a streaming server a moment to write its confirmation frame, so a + // caller that opens and immediately grades is not racing the first write. + if (state.open) { + await session.waitFor(() => true, options.openTimeoutMs ?? 2000); + } + return session; +} + +/** Read SSE `data:` frames until the stream ends or the request is aborted. */ +async function readFrames( + res: Response, + ingest: (frame: unknown) => void, + sseComments: string[] +): Promise { + if (!res.body) return; + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buffer = ''; + try { + for (;;) { + const { value, done } = await reader.read(); + if (done) break; + if (!value) continue; + buffer += decoder.decode(value, { stream: true }); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() ?? ''; + for (const line of lines) { + if (line.startsWith(':')) { + sseComments.push(line); + continue; + } + if (!line.startsWith('data:')) continue; + const payload = line.slice(5).trim(); + if (!payload) continue; + try { + ingest(JSON.parse(payload)); + } catch { + ingest(payload); + } + } + } + } catch { + // Aborted by cancel(), or the connection dropped. Either way the frames + // that arrived are what the scenario grades. + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} From 0adb7ce2eab686e1fd1d63af3f578ad3b55fbd05 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sun, 20 Sep 2026 19:34:09 +0000 Subject: [PATCH 04/27] feat(events): events-webhook, server conformance for subscription management The fourth of five scenarios: the 26 subscribe, TTL and unsubscribe rows. Delivery itself - signatures, the verification handshake, control envelopes - waits for events-webhook-delivery and a reachable callback. Two things shape it. A run holds one principal, so the key components the harness can vary (url, name, arguments) are graded and the principal half is reported untestable rather than passing a cross-tenant row that was never exercised across two tenants. And a server may cap concurrent subscriptions per principal: kitchen-sink allows two per event type, so every throwaway subscription is released as soon as its check is graded, including a probe the server was supposed to reject but accepted. Without that the scenario grades the cap error instead of the rule it was probing, which it did three times while being written. Every subscription is unsubscribed before the scenario returns, which matters against a public server. kitchen-sink at d2950655: 17/23, three divergences. It accepts an http:// callback URL where the document requires https and -32602 (two rows, one probe), and unsubscribing a key it never held succeeds instead of answering -32011 NotFound, so a client cannot tell teardown from a typo. metronome: 20/23, no divergences. The three failures are untestable rows: the unsupported-mode path (its only event type offers every mode), the auth requirement, and cross-tenant isolation. --- src/scenarios/index.ts | 2 + src/scenarios/server/events/webhook.ts | 1097 ++++++++++++++++++++++++ 2 files changed, 1099 insertions(+) create mode 100644 src/scenarios/server/events/webhook.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 58e3a1b7..4a479dc6 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -63,6 +63,7 @@ import { import { EventsDiscoveryScenario } from './server/events/discovery'; import { EventsPollScenario } from './server/events/poll'; import { EventsPushScenario } from './server/events/push'; +import { EventsWebhookScenario } from './server/events/webhook'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; @@ -240,6 +241,7 @@ const allClientScenariosList: ClientScenario[] = [ new EventsDiscoveryScenario(), new EventsPollScenario(), new EventsPushScenario(), + new EventsWebhookScenario(), // Prompts scenarios new PromptsListScenario(), diff --git a/src/scenarios/server/events/webhook.ts b/src/scenarios/server/events/webhook.ts new file mode 100644 index 00000000..4b61dff6 --- /dev/null +++ b/src/scenarios/server/events/webhook.ts @@ -0,0 +1,1097 @@ +/** + * MCP Events: webhook subscription management — `events/subscribe`, + * TTL negotiation, and `events/unsubscribe`. + * + * Scored against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events. Each check's verbatim + * excerpt lives beside its id in src/seps/sep-9999.yaml, where 9999 is a + * placeholder SEP number. + * + * This scenario deliberately stops at the subscription surface. Everything + * about what the server then POSTs — signatures, headers, the verification + * handshake, control envelopes — belongs to events-webhook-delivery, which + * needs a callback URL the server can reach. Here the callback is a URL that + * exists only to be a distinct key, so a server may well suspend delivery to + * it; that is not this scenario's business. + * + * Rows the principal owns are the awkward ones. The subscription key is + * `(principal, delivery.url, name, arguments)`, and a run holds exactly one + * principal: whatever credential the harness was pointed at, or none. So the + * three components the harness can vary are graded, and the principal + * component is reported untestable with the reason named, rather than passing + * a cross-tenant isolation row that was never exercised across two tenants. + * + * Every subscription this scenario creates is unsubscribed before it returns. + * Against a public server a leaked subscription is a server holding state for + * a callback nobody reads, so cleanup is part of the scenario rather than an + * afterthought. + */ + +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_CAPABILITY, + EVENTS_EXTENSION_ID, + EVENTS_NOT_FOUND, + EVENTS_SPEC_REF, + EVENTS_SUBSCRIBE_METHOD, + EVENTS_UNSUBSCRIBE_METHOD, + EVENTS_UNSUPPORTED, + JSONRPC_INVALID_PARAMS, + JSONRPC_METHOD_NOT_FOUND, + describeValue, + descriptorName, + eventsCheck, + eventsListAll, + firstSupporting, + isObject, + isValidCursor, + minimalArguments, + type EventDescriptor +} from './helpers'; + +/** A callback URL that is syntactically valid and points nowhere in use. */ +const CALLBACK_BASE = 'https://conformance.invalid/mcp-events'; + +/** A suggestion the server can plausibly grant, for the TTL rows. */ +const TTL_SUGGESTION_MS = 3600_000; + +const SUBSCRIBE_IDS = [ + 'sep-9999-subscribe-webhook-only', + 'sep-9999-subscribe-secret-required', + 'sep-9999-subscribe-secret-format', + 'sep-9999-subscribe-secret-rejected', + 'sep-9999-subscribe-url-https-required', + 'sep-9999-subscribe-url-non-https-rejected', + 'sep-9999-subscribe-auth-required', + 'sep-9999-subscribe-key-composition', + 'sep-9999-subscribe-key-immutable', + 'sep-9999-subscribe-idempotent-upsert', + 'sep-9999-subscribe-id-derived', + 'sep-9999-subscribe-id-not-an-input', + 'sep-9999-subscribe-refresh-replaces-secret', + 'sep-9999-subscribe-refresh-reactivates', + 'sep-9999-subscribe-response-cursor', + 'sep-9999-subscribe-response-truncated', + 'sep-9999-subscribe-cross-tenant-isolation' +] as const; + +const TTL_IDS = [ + 'sep-9999-ttl-refresh-before-lte-suggestion', + 'sep-9999-ttl-no-rejection-path', + 'sep-9999-ttl-null-only-when-requested', + 'sep-9999-ttl-omitted-means-default', + 'sep-9999-ttl-long-grant-retained', + 'sep-9999-ttl-no-expiry-persisted', + 'sep-9999-ttl-no-expiry-gc-terminated' +] as const; + +const UNSUBSCRIBE_IDS = [ + 'sep-9999-unsubscribe-by-key', + 'sep-9999-unsubscribe-unknown-not-found' +] as const; + +const ALL_IDS = [...SUBSCRIBE_IDS, ...TTL_IDS, ...UNSUBSCRIBE_IDS]; + +interface SubscribeResult { + id?: unknown; + refreshBefore?: unknown; + cursor?: unknown; + truncated?: unknown; + deliveryStatus?: unknown; +} + +/** A Standard Webhooks secret: `whsec_` plus base64 of 32 random bytes. */ +function freshSecret(): string { + const bytes = new Uint8Array(32); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = Math.floor(Math.random() * 256); + } + return `whsec_${Buffer.from(bytes).toString('base64')}`; +} + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) + ); +} + +export class EventsWebhookScenario implements ClientScenario { + name = 'events-webhook'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: webhook subscription management. + +**Methods**: \`events/subscribe\`, \`events/unsubscribe\`, plus \`events/list\` to discover an event type advertising \`webhook\` delivery + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-subscribe-secret-*\` — the \`whsec_\` Standard Webhooks secret is required, and a malformed one is rejected with \`InvalidParams\` +- \`sep-9999-subscribe-url-*\` — callback URLs must be https, and a non-https one is rejected +- \`sep-9999-subscribe-key-*\` / \`sep-9999-subscribe-id-*\` — the compound key, its immutability, the derived \`id\`, and that \`id\` is not an input +- \`sep-9999-subscribe-idempotent-upsert\` / \`sep-9999-subscribe-response-*\` — a repeat subscribe refreshes in place and returns \`cursor\` and \`truncated\` +- \`sep-9999-ttl-*\` — the grant is at or under the suggestion, there is no rejection path, and \`null\` comes back only when asked for +- \`sep-9999-unsubscribe-*\` — teardown by key, and \`-32011 NotFound\` for a key the server does not hold + +**Scope**: subscription management only. Signatures, the verification handshake and control envelopes belong to events-webhook-delivery, which needs a reachable callback. + +**The principal is one row's blind spot**: the key is \`(principal, delivery.url, name, arguments)\` and a run holds one principal, so cross-tenant isolation and the auth requirement report untestable rather than passing unexercised. + +**Cleanup**: every subscription created here is unsubscribed before the scenario returns.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + try { + const capabilities = await conn.discover(); + const caps = isObject(capabilities.capabilities) + ? capabilities.capabilities + : {}; + const declared = caps[EVENTS_CAPABILITY] !== undefined; + + const listed = await eventsListAll(conn); + if ('error' in listed) { + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no webhook-capable event type could be discovered. See the events-discovery scenario.` + ); + } + + const target = firstSupporting(listed.descriptors, 'webhook'); + const name = target ? descriptorName(target) : undefined; + if (!target || !name) { + return untestableAll( + ALL_IDS, + listed.descriptors.length === 0 + ? '`events/list` returned an empty catalog, so no webhook-capable event type could be exercised.' + : 'No event type advertises `webhook` delivery, so `events/subscribe` could not be exercised. Webhook is optional per event type.' + ); + } + + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema, so no subscription could be created.` + ); + } + + return await this.webhookChecks(conn, listed.descriptors, name, args); + } finally { + await conn.close(); + } + } + + private async webhookChecks( + conn: Connection, + descriptors: EventDescriptor[], + name: string, + args: Record + ): Promise { + const checks: ConformanceCheck[] = []; + const created: Array<{ url: string; name: string }> = []; + const url = `${CALLBACK_BASE}/${Date.now()}`; + + // A server may cap concurrent subscriptions per principal — kitchen-sink + // allows two per event type — and a run that holds every probe's + // subscription open hits that cap and then grades the cap error instead of + // the rule it was probing. So each throwaway subscription is released as + // soon as its check is graded, and only the one the later checks build on + // is held. + const release = async (url: string, subName: string): Promise => { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name: subName, + arguments: args, + delivery: { mode: 'webhook', url } + }); + } catch { + // Never created, or already gone. + } + const at = created.findIndex((c) => c.url === url && c.name === subName); + if (at >= 0) created.splice(at, 1); + }; + + const subscribe = async ( + params: Record + ): Promise<{ result: SubscribeResult } | { error: JsonRpcError }> => { + try { + const result = await conn.request( + EVENTS_SUBSCRIBE_METHOD, + params + ); + const delivery = isObject(params.delivery) ? params.delivery : {}; + if (typeof delivery.url === 'string') { + created.push({ url: delivery.url, name: String(params.name) }); + } + return { result: result ?? {} }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } + }; + + try { + // --- A subscription the rest of the scenario builds on ---------------- + const first = await subscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url, secret: freshSecret() }, + cursor: null, + ttlMs: TTL_SUGGESTION_MS + }); + + if ('error' in first) { + const err = first.error; + checks.push( + eventsCheck( + 'sep-9999-subscribe-webhook-only', + '`events/subscribe` is ONLY used for webhook delivery. Poll and push do not need it.', + err.code === JSONRPC_METHOD_NOT_FOUND ? 'FAILURE' : 'WARNING', + { + errorMessage: `Event type \`${name}\` advertises \`webhook\` delivery but a well-formed \`${EVENTS_SUBSCRIBE_METHOD}\` answered ${err.code} ${err.message}.`, + details: { code: err.code, message: err.message, data: err.data } + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-9999-subscribe-webhook-only'), + `No subscription could be created for \`${name}\` (${err.code} ${err.message}), so the subscription surface could not be exercised.` + ) + ); + return dedupe(checks); + } + + checks.push( + eventsCheck( + 'sep-9999-subscribe-webhook-only', + '`events/subscribe` is ONLY used for webhook delivery. Poll and push do not need it.', + 'SUCCESS', + { details: { name } } + ) + ); + + checks.push(...this.responseChecks(first.result)); + checks.push(...this.ttlChecks(first.result)); + checks.push( + ...(await this.ttlNegotiationChecks( + subscribe, + release, + name, + args, + url + )) + ); + checks.push( + ...(await this.rejectionChecks( + subscribe, + release, + name, + args, + descriptors + )) + ); + checks.push( + ...(await this.identityChecks( + subscribe, + release, + first.result, + name, + args, + url + )) + ); + checks.push( + ...(await this.unsubscribeChecks(conn, first.result, name, args, url)) + ); + return dedupe(checks); + } finally { + // Leave nothing behind, including on a public server. + for (const sub of created) { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name: sub.name, + arguments: args, + delivery: { mode: 'webhook', url: sub.url } + }); + } catch { + // Already gone, or never created. Nothing to do. + } + } + } + } + + /** `cursor` and `truncated` on the subscribe response. */ + private responseChecks(result: SubscribeResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + out.push( + isValidCursor(result.cursor) + ? eventsCheck( + 'sep-9999-subscribe-response-cursor', + "The subscribe response carries `cursor`, a safe-to-persist watermark that advances the client's cursor even if no events arrive before next refresh.", + 'SUCCESS', + { details: { cursor: result.cursor } } + ) + : eventsCheck( + 'sep-9999-subscribe-response-cursor', + "The subscribe response carries `cursor`, a safe-to-persist watermark that advances the client's cursor even if no events arrive before next refresh.", + 'FAILURE', + { + errorMessage: `\`cursor\` is ${describeValue(result.cursor)}, expected a string or null.` + } + ) + ); + out.push( + result.truncated === undefined || typeof result.truncated === 'boolean' + ? eventsCheck( + 'sep-9999-subscribe-response-truncated', + 'The subscribe response carries `truncated`, true if delivery started later than the supplied cursor.', + 'SUCCESS', + { details: { truncated: result.truncated ?? false } } + ) + : eventsCheck( + 'sep-9999-subscribe-response-truncated', + 'The subscribe response carries `truncated`, true if delivery started later than the supplied cursor.', + 'FAILURE', + { + errorMessage: `\`truncated\` is ${describeValue(result.truncated)}, expected a boolean.` + } + ) + ); + return out; + } + + /** The grant on the subscription the scenario opened with a suggestion. */ + private ttlChecks(result: SubscribeResult): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const grant = result.refreshBefore; + const description = + '`refreshBefore` (response) is the grant. It SHOULD be less than or equal to the suggestion.'; + + if (grant === null) { + out.push( + eventsCheck( + 'sep-9999-ttl-null-only-when-requested', + 'A server MUST NOT return `null` unless the client suggested `ttlMs: null` — no expiry exceeds every finite suggestion.', + 'FAILURE', + { + errorMessage: `The subscribe carried \`ttlMs: ${TTL_SUGGESTION_MS}\` and the server granted \`refreshBefore: null\`, which is no expiry.` + } + ) + ); + out.push( + untestableCheck( + 'sep-9999-ttl-refresh-before-lte-suggestion', + 'sep-9999-ttl-refresh-before-lte-suggestion', + description, + 'The server granted no expiry, so there is no finite grant to compare against the suggestion.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + return out; + } + + out.push( + eventsCheck( + 'sep-9999-ttl-null-only-when-requested', + 'A server MUST NOT return `null` unless the client suggested `ttlMs: null` — no expiry exceeds every finite suggestion.', + 'SUCCESS', + { details: { refreshBefore: grant } } + ) + ); + + const grantedAt = typeof grant === 'string' ? Date.parse(grant) : NaN; + if (!Number.isFinite(grantedAt)) { + out.push( + eventsCheck( + 'sep-9999-ttl-refresh-before-lte-suggestion', + description, + 'FAILURE', + { + errorMessage: `\`refreshBefore\` is ${describeValue(grant)}, expected an ISO 8601 timestamp or null.` + } + ) + ); + return out; + } + + // A minute of slack: the suggestion is measured from when the harness + // sent the request, the grant from when the server handled it. + const ceiling = Date.now() + TTL_SUGGESTION_MS + 60_000; + out.push( + grantedAt <= ceiling + ? eventsCheck( + 'sep-9999-ttl-refresh-before-lte-suggestion', + description, + 'SUCCESS', + { + details: { refreshBefore: grant, suggestedMs: TTL_SUGGESTION_MS } + } + ) + : eventsCheck( + 'sep-9999-ttl-refresh-before-lte-suggestion', + description, + 'WARNING', + { + errorMessage: `Granted \`refreshBefore\` is ${grant}, about ${Math.round((grantedAt - Date.now()) / 1000)}s out, past the suggested ${TTL_SUGGESTION_MS / 1000}s.` + } + ) + ); + return out; + } + + /** Omitted, absurd and no-expiry TTL suggestions. */ + private async ttlNegotiationChecks( + subscribe: ( + p: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, + release: (url: string, name: string) => Promise, + name: string, + args: Record, + baseUrl: string + ): Promise { + const out: ConformanceCheck[] = []; + + // Omitting ttlMs means "server default", which is still a grant. + const omitted = await subscribe({ + name, + arguments: args, + delivery: { + mode: 'webhook', + url: `${baseUrl}-ttl-default`, + secret: freshSecret() + }, + cursor: null + }); + await release(`${baseUrl}-ttl-default`, name); + out.push( + 'error' in omitted + ? eventsCheck( + 'sep-9999-ttl-omitted-means-default', + 'Omitting `ttlMs` means "server default."', + 'FAILURE', + { + errorMessage: `A subscribe with no \`ttlMs\` answered ${omitted.error.code} ${omitted.error.message}; omitting it is defined as asking for the server default.` + } + ) + : omitted.result.refreshBefore === null + ? eventsCheck( + 'sep-9999-ttl-omitted-means-default', + 'Omitting `ttlMs` means "server default." An explicit `ttlMs: null` requests a subscription with no expiry.', + 'FAILURE', + { + errorMessage: + 'A subscribe with no `ttlMs` was granted `refreshBefore: null`, which only an explicit `ttlMs: null` may request.' + } + ) + : eventsCheck( + 'sep-9999-ttl-omitted-means-default', + 'Omitting `ttlMs` means "server default."', + 'SUCCESS', + { details: { refreshBefore: omitted.result.refreshBefore } } + ) + ); + + // Clamping is self-announcing in both directions, so neither an + // impractically short nor a very long suggestion is an error. + const short = await subscribe({ + name, + arguments: args, + delivery: { + mode: 'webhook', + url: `${baseUrl}-ttl-short`, + secret: freshSecret() + }, + cursor: null, + ttlMs: 1000 + }); + // Released before the next one is opened: a server may cap concurrent + // subscriptions per principal, and the primary subscription holds a slot. + await release(`${baseUrl}-ttl-short`, name); + const long = await subscribe({ + name, + arguments: args, + delivery: { + mode: 'webhook', + url: `${baseUrl}-ttl-long`, + secret: freshSecret() + }, + cursor: null, + ttlMs: 30 * 24 * 3600_000 + }); + await release(`${baseUrl}-ttl-long`, name); + const rejected = [ + ['1000ms', short], + ['30 days', long] + ].filter(([, r]) => 'error' in (r as object)) as Array< + [string, { error: JsonRpcError }] + >; + out.push( + rejected.length === 0 + ? eventsCheck( + 'sep-9999-ttl-no-rejection-path', + 'Clamping in either direction is self-announcing, so a clamped grant is not an error and there is no rejection path for TTL values.', + 'SUCCESS', + { + details: { + short: + 'error' in short ? undefined : short.result.refreshBefore, + long: 'error' in long ? undefined : long.result.refreshBefore + } + } + ) + : eventsCheck( + 'sep-9999-ttl-no-rejection-path', + 'Clamping in either direction is self-announcing, so a clamped grant is not an error and there is no rejection path for TTL values.', + 'FAILURE', + { + errorMessage: rejected + .map( + ([label, r]) => + `\`ttlMs\` of ${label} was rejected with ${r.error.code} ${r.error.message}` + ) + .join('; ') + } + ) + ); + + // The durability rows need a restart, which the harness cannot ask for. + out.push( + ...untestableAll( + [ + 'sep-9999-ttl-long-grant-retained', + 'sep-9999-ttl-no-expiry-persisted', + 'sep-9999-ttl-no-expiry-gc-terminated' + ], + 'Grading retention across a restart needs the server under test to be restarted mid-run, which the harness cannot do over the wire.', + 'WARNING' + ) + ); + + return out; + } + + /** What a malformed subscribe must be rejected with. */ + private async rejectionChecks( + subscribe: ( + p: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, + release: (url: string, name: string) => Promise, + name: string, + args: Record, + descriptors: EventDescriptor[] + ): Promise { + const out: ConformanceCheck[] = []; + + const expectInvalidParams = async ( + id: string, + description: string, + params: Record, + what: string + ): Promise => { + const res = await subscribe(params); + if (!('error' in res)) { + // A probe the server was supposed to reject but accepted has just + // taken a subscription slot, and holding it would make the next probe + // grade a cap error instead of its own rule. + const delivery = isObject(params.delivery) ? params.delivery : {}; + if (typeof delivery.url === 'string') { + await release(delivery.url, String(params.name)); + } + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `${what} was accepted; the document requires \`-32602 InvalidParams\`.`, + details: { result: res.result } + }); + } + return res.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck(id, description, 'SUCCESS', { + details: { code: res.error.code } + }) + : eventsCheck(id, description, 'WARNING', { + errorMessage: `${what} was rejected with ${res.error.code} ${res.error.message}, where the document names \`-32602 InvalidParams\`.`, + details: { error: res.error } + }); + }; + + const base = { + name, + arguments: args, + cursor: null + }; + + out.push( + await expectInvalidParams( + 'sep-9999-subscribe-secret-required', + '`delivery.secret` is REQUIRED. The client supplies the HMAC signing secret; the server never generates one.', + { + ...base, + delivery: { mode: 'webhook', url: `${CALLBACK_BASE}/no-secret` } + }, + 'A subscribe with no `delivery.secret`' + ) + ); + + out.push( + await expectInvalidParams( + 'sep-9999-subscribe-secret-format', + 'The value MUST be a Standard Webhooks symmetric secret: the literal prefix `whsec_` followed by base64 of 24–64 random bytes.', + { + ...base, + delivery: { + mode: 'webhook', + url: `${CALLBACK_BASE}/bad-prefix`, + secret: 'not-a-standard-webhooks-secret' + } + }, + 'A `delivery.secret` without the `whsec_` prefix' + ) + ); + + // Right prefix, too few bytes: 8 decoded, where the floor is 24. + out.push( + await expectInvalidParams( + 'sep-9999-subscribe-secret-rejected', + 'Servers MUST reject a `delivery.secret` that is not `whsec_` followed by base64 decoding to 24–64 bytes with `InvalidParams`.', + { + ...base, + delivery: { + mode: 'webhook', + url: `${CALLBACK_BASE}/short-secret`, + secret: `whsec_${Buffer.from(new Uint8Array(8)).toString('base64')}` + } + }, + 'A `whsec_` secret decoding to 8 bytes, under the 24-byte floor,' + ) + ); + + const nonHttps = await expectInvalidParams( + 'sep-9999-subscribe-url-non-https-rejected', + 'Servers MUST reject `events/subscribe` with a non-`https` `delivery.url` (`-32602 InvalidParams`).', + { + ...base, + delivery: { + mode: 'webhook', + url: 'http://conformance.invalid/insecure', + secret: freshSecret() + } + }, + 'A subscribe with an `http://` `delivery.url`' + ); + out.push(nonHttps); + // The requirement and its enforcement are one probe; report both so the + // manifest does not carry an untested row for the rule itself. + out.push( + eventsCheck( + 'sep-9999-subscribe-url-https-required', + 'Callback URLs MUST use `https://`.', + nonHttps.status, + { + errorMessage: nonHttps.errorMessage, + details: { gradedBy: 'sep-9999-subscribe-url-non-https-rejected' } + } + ) + ); + + // A type that does not offer webhook, when the catalog has one. + const nonWebhook = descriptors.find((d) => { + const modes = Array.isArray(d.delivery) ? d.delivery : []; + return modes.length > 0 && !modes.includes('webhook'); + }); + const nonWebhookName = nonWebhook ? descriptorName(nonWebhook) : undefined; + if (!nonWebhookName) { + out.push( + untestableCheck( + 'sep-9999-error-unsupported', + 'sep-9999-error-unsupported', + '`-32014 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'Every event type the server offers advertises `webhook` delivery, so there is no type to probe the unsupported-mode path with.', + [EVENTS_SPEC_REF] + ) + ); + } else { + const res = await subscribe({ + name: nonWebhookName, + arguments: {}, + delivery: { + mode: 'webhook', + url: `${CALLBACK_BASE}/unsupported`, + secret: freshSecret() + }, + cursor: null + }); + out.push( + !('error' in res) + ? eventsCheck( + 'sep-9999-error-unsupported', + '`-32014 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'FAILURE', + { + errorMessage: `Event type \`${nonWebhookName}\` does not advertise \`webhook\` delivery but \`${EVENTS_SUBSCRIBE_METHOD}\` returned a subscription.` + } + ) + : res.error.code === EVENTS_UNSUPPORTED + ? eventsCheck( + 'sep-9999-error-unsupported', + '`-32014 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'SUCCESS', + { details: { code: res.error.code } } + ) + : eventsCheck( + 'sep-9999-error-unsupported', + '`-32014 Unsupported` — the request is well-formed but a requested capability or option is not supported here.', + 'WARNING', + { + errorMessage: `Subscribing to \`${nonWebhookName}\`, which offers ${JSON.stringify(nonWebhook?.delivery)}, answered ${res.error.code} ${res.error.message} rather than \`-32014 Unsupported\`.` + } + ) + ); + } + + // The principal half of the surface. One run, one credential. + out.push( + untestableCheck( + 'sep-9999-subscribe-auth-required', + 'sep-9999-subscribe-auth-required', + '`events/subscribe` and `events/unsubscribe` MUST be called with an authenticated principal; servers MUST reject calls without one with `-32012 Forbidden`.', + 'The harness sends whatever credential it was pointed at, on every request, and has no way to make the same call as an anonymous caller. Grading this needs a runner that can drop its own auth for one probe.', + [EVENTS_SPEC_REF] + ) + ); + + return out; + } + + /** The compound key, the derived id, and what a refresh does. */ + private async identityChecks( + subscribe: ( + p: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, + release: (url: string, name: string) => Promise, + first: SubscribeResult, + name: string, + args: Record, + url: string + ): Promise { + const out: ConformanceCheck[] = []; + const firstId = first.id; + + out.push( + typeof firstId === 'string' && firstId.length > 0 + ? eventsCheck( + 'sep-9999-subscribe-id-derived', + 'The server computes a deterministic `id` over the key and returns it in the subscribe response. It is stable across refreshes and server restarts.', + 'SUCCESS', + { details: { id: firstId } } + ) + : eventsCheck( + 'sep-9999-subscribe-id-derived', + 'The server computes a deterministic `id` over the key and returns it in the subscribe response.', + 'FAILURE', + { + errorMessage: `The subscribe response carried \`id\` ${describeValue(firstId)}, expected a string.` + } + ) + ); + + // Same key again: same id, and the TTL is re-granted in place. + const again = await subscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url, secret: freshSecret() }, + cursor: null, + ttlMs: TTL_SUGGESTION_MS + }); + if ('error' in again) { + out.push( + eventsCheck( + 'sep-9999-subscribe-idempotent-upsert', + '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields.', + 'FAILURE', + { + errorMessage: `A second subscribe with the same key answered ${again.error.code} ${again.error.message}; the call is defined as an idempotent upsert.` + } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-subscribe-key-composition', + 'sep-9999-subscribe-key-immutable', + 'sep-9999-subscribe-refresh-replaces-secret' + ], + 'The repeat subscribe failed, so nothing about key identity could be compared.' + ) + ); + } else { + out.push( + again.result.id === firstId + ? eventsCheck( + 'sep-9999-subscribe-idempotent-upsert', + '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields in place.', + 'SUCCESS', + { details: { id: firstId } } + ) + : eventsCheck( + 'sep-9999-subscribe-idempotent-upsert', + '`events/subscribe` is idempotent — calling it again with the same subscription key refreshes the TTL and updates mutable fields in place.', + 'FAILURE', + { + errorMessage: `The same key produced a different \`id\` (${describeValue(firstId)} then ${describeValue(again.result.id)}), so the second call created a second subscription.` + } + ) + ); + + // A refresh carrying a new secret is accepted; whether the next delivery + // is signed with it belongs to events-webhook-delivery. + out.push( + untestableCheck( + 'sep-9999-subscribe-refresh-replaces-secret', + 'sep-9999-subscribe-refresh-replaces-secret', + 'On an idempotent subscribe against an existing key, `delivery.secret` is replaced.', + 'The refresh carrying a new secret was accepted, but confirming the replacement needs a delivery signed with it, which events-webhook-delivery covers.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + // Vary one component at a time: a different url, then different arguments. + const otherUrl = await subscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url: `${url}-other`, secret: freshSecret() }, + cursor: null + }); + await release(`${url}-other`, name); + const distinctIds = + !('error' in otherUrl) && + typeof otherUrl.result.id === 'string' && + otherUrl.result.id !== firstId; + out.push( + 'error' in otherUrl + ? untestableCheck( + 'sep-9999-subscribe-key-composition', + 'sep-9999-subscribe-key-composition', + 'The subscription key is `(principal, delivery.url, name, arguments)`.', + `A subscribe differing only in \`delivery.url\` answered ${otherUrl.error.code} ${otherUrl.error.message}, so the two keys could not be compared.`, + [EVENTS_SPEC_REF] + ) + : distinctIds + ? eventsCheck( + 'sep-9999-subscribe-key-composition', + 'The subscription key is `(principal, delivery.url, name, arguments)`; a call differing in any component addresses a different subscription.', + 'SUCCESS', + { + details: { + note: 'Graded on `delivery.url`; the `principal` component needs a second tenant.', + ids: [firstId, otherUrl.result.id] + } + } + ) + : eventsCheck( + 'sep-9999-subscribe-key-composition', + 'The subscription key is `(principal, delivery.url, name, arguments)`; a call differing in any component addresses a different subscription.', + 'FAILURE', + { + errorMessage: `Two subscriptions differing in \`delivery.url\` share the id ${describeValue(otherUrl.result.id)}, so the URL is not part of the key.` + } + ) + ); + out.push( + 'error' in otherUrl + ? untestableCheck( + 'sep-9999-subscribe-key-immutable', + 'sep-9999-subscribe-key-immutable', + "All four key components are immutable for the subscription's lifetime.", + 'The second subscription could not be created, so immutability could not be observed.', + [EVENTS_SPEC_REF] + ) + : eventsCheck( + 'sep-9999-subscribe-key-immutable', + "All four components are immutable for the subscription's lifetime: a subscribe call with a different value for any of them addresses a different subscription.", + distinctIds ? 'SUCCESS' : 'FAILURE', + { + errorMessage: distinctIds + ? undefined + : 'Changing `delivery.url` did not address a different subscription, so a key component was mutated in place.' + } + ) + ); + + // `id` is a routing handle, never an input. + const byId = await subscribe({ + name, + arguments: args, + id: firstId, + delivery: { mode: 'webhook', url: `${url}-by-id`, secret: freshSecret() }, + cursor: null + }); + await release(`${url}-by-id`, name); + out.push( + 'error' in byId + ? eventsCheck( + 'sep-9999-subscribe-id-not-an-input', + "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method.", + 'SUCCESS', + { + details: { + note: 'The server rejected a subscribe carrying `id`.', + code: byId.error.code + } + } + ) + : byId.result.id === firstId + ? eventsCheck( + 'sep-9999-subscribe-id-not-an-input', + "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method.", + 'FAILURE', + { + errorMessage: `A subscribe carrying \`id: ${String(firstId)}\` with a different \`delivery.url\` returned that same id, so \`id\` addressed the subscription instead of the key.` + } + ) + : eventsCheck( + 'sep-9999-subscribe-id-not-an-input', + "A caller who learns another tenant's derived `id` gains nothing — `id` is not accepted as input to any method.", + 'SUCCESS', + { + details: { + note: 'The supplied `id` was ignored; the key decided the subscription.', + ids: [firstId, byId.result.id] + } + } + ) + ); + + out.push( + untestableCheck( + 'sep-9999-subscribe-cross-tenant-isolation', + 'sep-9999-subscribe-cross-tenant-isolation', + 'Because the key includes `principal` and `delivery.url`, two distinct tenants subscribing to the same `(name, arguments)` get distinct subscriptions.', + 'A run holds one principal, so the two-tenant case cannot be constructed. The `delivery.url` half of the same rule is graded by sep-9999-subscribe-key-composition.', + [EVENTS_SPEC_REF] + ) + ); + + out.push( + untestableCheck( + 'sep-9999-subscribe-refresh-reactivates', + 'sep-9999-subscribe-refresh-reactivates', + 'On an idempotent subscribe against an existing key, `active` is set to `true` and suspended delivery resumes.', + 'Suspension is observable only through the OPTIONAL `deliveryStatus` object, and reaching it needs sustained delivery failure against a callback the harness controls.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + + return out; + } + + /** Teardown by key, and what an unknown key answers. */ + private async unsubscribeChecks( + conn: Connection, + first: SubscribeResult, + name: string, + args: Record, + url: string + ): Promise { + const out: ConformanceCheck[] = []; + const unsubscribe = async ( + params: Record + ): Promise<{ ok: true } | { error: JsonRpcError }> => { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, params); + return { ok: true }; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } + }; + + const byKey = await unsubscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url } + }); + out.push( + 'error' in byKey + ? eventsCheck( + 'sep-9999-unsubscribe-by-key', + '`events/unsubscribe` is eager cleanup; the server looks the subscription up by the same compound key used for idempotent upsert on subscribe.', + 'FAILURE', + { + errorMessage: `Unsubscribing the subscription just created (id ${describeValue(first.id)}) by its key answered ${byKey.error.code} ${byKey.error.message}.` + } + ) + : eventsCheck( + 'sep-9999-unsubscribe-by-key', + '`events/unsubscribe` is eager cleanup; the server looks the subscription up by the same compound key used for idempotent upsert on subscribe.', + 'SUCCESS', + { details: { id: first.id } } + ) + ); + + // A key the server cannot hold: never subscribed, and now also the key + // that was just torn down. + const unknown = await unsubscribe({ + name, + arguments: args, + delivery: { mode: 'webhook', url: `${CALLBACK_BASE}/never-subscribed` } + }); + out.push( + 'error' in unknown + ? unknown.error.code === EVENTS_NOT_FOUND + ? eventsCheck( + 'sep-9999-unsubscribe-unknown-not-found', + 'No subscription matching the key on `events/unsubscribe` returns `-32011 NotFound`.', + 'SUCCESS', + { details: { code: unknown.error.code } } + ) + : eventsCheck( + 'sep-9999-unsubscribe-unknown-not-found', + 'No subscription matching the key on `events/unsubscribe` returns `-32011 NotFound`.', + 'WARNING', + { + errorMessage: `An unknown subscription key answered ${unknown.error.code} ${unknown.error.message}, where the document names \`-32011 NotFound\`.` + } + ) + : eventsCheck( + 'sep-9999-unsubscribe-unknown-not-found', + 'No subscription matching the key on `events/unsubscribe` returns `-32011 NotFound`.', + 'FAILURE', + { + errorMessage: + 'Unsubscribing a key that was never subscribed succeeded. A client cannot tell teardown from a no-op, and a typo in the key reads as success.' + } + ) + ); + + return out; + } +} + +/** Keep the first check emitted per id, so a fallback path cannot double-report. */ +function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { + const seen = new Set(); + return checks.filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }); +} From 6d76f934ae8944d3bffdba4ad05b33d1818f47ea Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sun, 20 Sep 2026 19:43:01 +0000 Subject: [PATCH 05/27] feat(events): events-webhook-delivery, the fifth and last phase-1 scenario Grades what the server POSTs: the verification handshake, Standard Webhooks signing, retries, control envelopes and the SSRF rules. receiver.ts runs a real HTTP endpoint and keeps the raw body bytes, because the signature covers those and re-serializing the JSON would change them. The scenario needs a callback the server can reach, and EVENTS_WEBHOOK_CALLBACK_BASE supplies one. Without it the receiver binds to loopback, which is not a degraded mode but a different question: the document requires a server to refuse a non-routable callback, so a loopback URL is the SSRF probe. A server that refuses it passes the SSRF rows and reports delivery untestable; one that POSTs to 127.0.0.1 fails them and hands over real deliveries to grade everything else against. kitchen-sink at d2950655: 12/21, four divergences, two of them security-shaped. It accepts and delivers to a loopback callback, so a caller can aim deliveries at anything the server can reach and the caller cannot. It signs with the literal whsec_ string as the HMAC key, where the document says the key is the base64-decoded bytes after that prefix. No Standard Webhooks receiver verifies these signatures; confirmed by hand against four candidate formulas before filing it here. It delivers without the verification handshake, so an unverified third-party URL receives POSTs. metronome refuses the loopback callback outright, which passes the SSRF rows and leaves the delivery rows untestable. Grading it needs a public callback. One harness bug, found while reading the results: the retry row folded signature validity into its freshness check, so a server with the wrong formula was reported as reusing timestamps it had not reused. Freshness is now graded on distinct timestamps and signatures alone. Coverage is now 117 of the 131 declared rows across the five scenarios. The remaining 14 need a server that changes at runtime (schema evolution, list_changed, event-type removal) or error paths this run did not reach. --- src/scenarios/index.ts | 2 + src/scenarios/server/events/receiver.ts | 171 +++ .../server/events/webhook-delivery.ts | 1128 +++++++++++++++++ 3 files changed, 1301 insertions(+) create mode 100644 src/scenarios/server/events/receiver.ts create mode 100644 src/scenarios/server/events/webhook-delivery.ts diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 4a479dc6..82899e06 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -64,6 +64,7 @@ import { EventsDiscoveryScenario } from './server/events/discovery'; import { EventsPollScenario } from './server/events/poll'; import { EventsPushScenario } from './server/events/push'; import { EventsWebhookScenario } from './server/events/webhook'; +import { EventsWebhookDeliveryScenario } from './server/events/webhook-delivery'; import { SkillsDirectoryReadScenario } from './server/skills/directory'; import { SkillsEnumerationScenario } from './server/skills/enumeration'; import { SkillsManifestScenario } from './server/skills/manifest'; @@ -242,6 +243,7 @@ const allClientScenariosList: ClientScenario[] = [ new EventsPollScenario(), new EventsPushScenario(), new EventsWebhookScenario(), + new EventsWebhookDeliveryScenario(), // Prompts scenarios new PromptsListScenario(), diff --git a/src/scenarios/server/events/receiver.ts b/src/scenarios/server/events/receiver.ts new file mode 100644 index 00000000..7c966c07 --- /dev/null +++ b/src/scenarios/server/events/receiver.ts @@ -0,0 +1,171 @@ +/** + * A callback endpoint for the webhook-delivery scenario. + * + * The scenario needs to be the receiver, not just the subscriber: the + * signature, the Standard Webhooks headers, the verification challenge and the + * retry behaviour are all only observable from the endpoint the server POSTs + * to. So this runs a real HTTP server and records every request verbatim, + * including the raw body bytes, because the signature is computed over those + * and re-serializing the JSON would change them. + * + * Per-path behaviour lets one receiver serve every probe: a path that echoes + * the challenge and accepts, one that redirects, one that refuses permanently, + * and one that fails a few times before accepting. + */ + +import http from 'node:http'; +import type { AddressInfo } from 'node:net'; + +export interface ReceivedDelivery { + path: string; + method: string; + headers: Record; + /** The body exactly as it arrived, which is what the signature covers. */ + rawBody: string; + /** Parsed body, when it was JSON. */ + json?: Record; + /** Milliseconds since the receiver started. */ + atMs: number; + /** What this receiver answered. */ + respondedStatus: number; +} + +export type PathBehaviour = + | { kind: 'accept' } + | { kind: 'redirect'; to: string } + | { kind: 'gone' } + | { kind: 'too-large' } + | { kind: 'fail-then-accept'; failures: number; status: number } + | { kind: 'wrong-challenge' }; + +export interface Receiver { + /** Base URL of the receiver, e.g. `http://127.0.0.1:53211`. */ + readonly url: string; + readonly deliveries: ReceivedDelivery[]; + /** Deliveries on one path, in arrival order. */ + on(path: string): ReceivedDelivery[]; + /** Set how a path answers. Unknown paths accept. */ + behave(path: string, behaviour: PathBehaviour): void; + /** Resolve once a delivery on `path` matches, or undefined at the deadline. */ + waitFor( + path: string, + predicate: (d: ReceivedDelivery) => boolean, + timeoutMs: number + ): Promise; + close(): Promise; +} + +export async function startReceiver(host = '127.0.0.1'): Promise { + const deliveries: ReceivedDelivery[] = []; + const behaviours = new Map(); + const failureCounts = new Map(); + const startedAt = Date.now(); + + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on('data', (c: Buffer) => chunks.push(c)); + req.on('end', () => { + const rawBody = Buffer.concat(chunks).toString('utf8'); + const path = (req.url ?? '/').split('?')[0]; + let json: Record | undefined; + try { + const parsed: unknown = JSON.parse(rawBody); + if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) { + json = parsed as Record; + } + } catch { + // Not JSON, which is itself something the scenario grades. + } + + const behaviour = behaviours.get(path) ?? { kind: 'accept' }; + const respond = (status: number, body?: string): void => { + deliveries.push({ + path, + method: req.method ?? 'GET', + headers: Object.fromEntries( + Object.entries(req.headers).map(([k, v]) => [ + k.toLowerCase(), + Array.isArray(v) ? v.join(', ') : (v ?? '') + ]) + ), + rawBody, + json, + atMs: Date.now() - startedAt, + respondedStatus: status + }); + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(body ?? '{}'); + }; + + switch (behaviour.kind) { + case 'redirect': + res.writeHead(302, { location: behaviour.to }); + deliveries.push({ + path, + method: req.method ?? 'GET', + headers: {}, + rawBody, + json, + atMs: Date.now() - startedAt, + respondedStatus: 302 + }); + res.end(); + return; + case 'gone': + respond(410); + return; + case 'too-large': + respond(413); + return; + case 'wrong-challenge': + respond(200, JSON.stringify({ challenge: 'not-the-nonce' })); + return; + case 'fail-then-accept': { + const seen = failureCounts.get(path) ?? 0; + if (seen < behaviour.failures) { + failureCounts.set(path, seen + 1); + respond(behaviour.status); + return; + } + break; + } + default: + break; + } + + // The verification handshake: prove intent by echoing the nonce. + const challenge = json?.challenge; + if (typeof challenge === 'string') { + respond(200, JSON.stringify({ challenge })); + return; + } + respond(200); + }); + }); + + await new Promise((resolve) => server.listen(0, host, resolve)); + const { port } = server.address() as AddressInfo; + + return { + url: `http://${host}:${port}`, + deliveries, + on(path) { + return deliveries.filter((d) => d.path === path); + }, + behave(path, behaviour) { + behaviours.set(path, behaviour); + }, + async waitFor(path, predicate, timeoutMs) { + const deadline = Date.now() + timeoutMs; + for (;;) { + const hit = deliveries.filter((d) => d.path === path).find(predicate); + if (hit) return hit; + if (Date.now() >= deadline) return undefined; + await new Promise((resolve) => setTimeout(resolve, 100)); + } + }, + async close() { + await new Promise((resolve) => server.close(() => resolve())); + } + }; +} diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts new file mode 100644 index 00000000..38149287 --- /dev/null +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -0,0 +1,1128 @@ +/** + * MCP Events: what the server POSTs to a webhook callback — the verification + * handshake, Standard Webhooks signing, retry behaviour, control envelopes and + * the SSRF rules. + * + * Scored against the merged design sketch on `main` of + * modelcontextprotocol/experimental-ext-triggers-events. Each check's verbatim + * excerpt lives beside its id in src/seps/sep-9999.yaml, where 9999 is a + * placeholder SEP number. + * + * **This scenario needs a callback the server under test can reach.** Set + * `EVENTS_WEBHOOK_CALLBACK_BASE` to a public https base URL forwarding to this + * harness (a tunnel, say), and the receiver binds to `0.0.0.0` behind it. + * + * With no such URL the scenario still runs, pointed at a loopback receiver, + * and that is not a degraded mode so much as a different question. The + * document requires a server to refuse a callback whose resolved IP is not + * globally routable, so a loopback URL is exactly the SSRF probe: a server + * that refuses it passes the SSRF rows and reports the delivery rows as + * untestable, while one that happily POSTs to 127.0.0.1 fails the SSRF rows + * and hands the harness real deliveries to grade everything else against. Both + * outcomes are informative, and neither is a false green. + * + * The signature is verified over the raw bytes, per the document's own + * instruction to receivers, so the scenario cannot accidentally pass a server + * that signs a re-serialization of its own JSON. + */ + +import { createHmac, timingSafeEqual } from 'node:crypto'; +import { ClientScenario, ConformanceCheck } from '../../../types'; +import type { Connection, RunContext } from '../../../connection'; +import { JsonRpcError } from '../../../connection'; +import { untestableCheck } from '../../untestable'; +import { + EVENTS_CAPABILITY, + EVENTS_CALLBACK_ENDPOINT_ERROR, + EVENTS_EXTENSION_ID, + EVENTS_SPEC_REF, + EVENTS_SUBSCRIBE_METHOD, + EVENTS_UNSUBSCRIBE_METHOD, + JSONRPC_METHOD_NOT_FOUND, + describeValue, + descriptorName, + eventsCheck, + eventsListAll, + firstSupporting, + isIso8601, + isObject, + minimalArguments +} from './helpers'; +import { + startReceiver, + type ReceivedDelivery, + type Receiver +} from './receiver'; + +/** How long to wait for the server to deliver something. */ +const DELIVERY_WAIT_MS = Number(process.env.EVENTS_DELIVERY_WAIT_MS ?? 20000); + +/** A public base URL forwarding to this harness, when one exists. */ +const PUBLIC_BASE = process.env.EVENTS_WEBHOOK_CALLBACK_BASE; + +/** 256 KiB, the body-size ceiling the document asks servers to respect. */ +const BODY_CEILING_BYTES = 256 * 1024; + +const DELIVERY_IDS = [ + 'sep-9999-delivery-post-json', + 'sep-9999-delivery-standard-webhooks-headers', + 'sep-9999-delivery-subscription-id-header', + 'sep-9999-delivery-signature-formula', + 'sep-9999-delivery-retry-regenerates-signature', + 'sep-9999-delivery-dual-sign-on-rotation', + 'sep-9999-delivery-body-size', + 'sep-9999-delivery-413-non-retryable', + 'sep-9999-delivery-410-non-retryable', + 'sep-9999-delivery-retries-bounded', + 'sep-9999-delivery-status-last-error-category' +] as const; + +const VERIFICATION_IDS = [ + 'sep-9999-verification-required-before-delivery', + 'sep-9999-verification-challenge-echo', + 'sep-9999-verification-failure-error', + 'sep-9999-verification-cached-per-principal-url', + 'sep-9999-verification-persisted-for-no-expiry', + 'sep-9999-verification-uses-ssrf-hardened-path', + 'sep-9999-verification-no-raw-endpoint-responses', + 'sep-9999-server-identity-key-discovery' +] as const; + +const SSRF_IDS = [ + 'sep-9999-ssrf-validate-callback-url', + 'sep-9999-ssrf-reject-non-routable', + 'sep-9999-ssrf-validate-at-delivery-time', + 'sep-9999-ssrf-no-redirects' +] as const; + +const ENVELOPE_IDS = [ + 'sep-9999-envelope-type-discriminator', + 'sep-9999-envelope-signed-like-deliveries', + 'sep-9999-envelope-webhook-id-format', + 'sep-9999-envelope-gap', + 'sep-9999-envelope-terminated' +] as const; + +const ALL_IDS = [ + ...DELIVERY_IDS, + ...VERIFICATION_IDS, + ...SSRF_IDS, + ...ENVELOPE_IDS +]; + +function untestableAll( + ids: readonly string[], + reason: string, + severity: 'FAILURE' | 'WARNING' = 'FAILURE' +): ConformanceCheck[] { + return ids.map((id) => + untestableCheck(id, id, id, reason, [EVENTS_SPEC_REF], severity) + ); +} + +function skipAll(reason: string): ConformanceCheck[] { + return ALL_IDS.map((id) => + eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) + ); +} + +function freshSecret(): { value: string; bytes: Buffer } { + const bytes = Buffer.alloc(32); + for (let i = 0; i < bytes.length; i++) { + bytes[i] = Math.floor(Math.random() * 256); + } + return { value: `whsec_${bytes.toString('base64')}`, bytes }; +} + +/** `HMAC-SHA256(secret, id + "." + timestamp + "." + body)`, base64, `v1,`. */ +function expectedSignature( + secret: Buffer, + webhookId: string, + timestamp: string, + rawBody: string +): string { + const mac = createHmac('sha256', secret) + .update(`${webhookId}.${timestamp}.${rawBody}`) + .digest('base64'); + return `v1,${mac}`; +} + +function signatureMatches(header: string, expected: string): boolean { + // The header may carry several space-delimited signatures during rotation. + return header + .split(/\s+/) + .filter(Boolean) + .some((candidate) => { + const a = Buffer.from(candidate); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); + }); +} + +export class EventsWebhookDeliveryScenario implements ClientScenario { + name = 'events-webhook-delivery'; + readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; + description = `MCP Events: webhook delivery, signing, verification and the SSRF rules. + +**Methods**: \`events/subscribe\` and \`events/unsubscribe\`, plus an HTTP receiver the harness runs and the server POSTs to + +**Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): + +- \`sep-9999-verification-*\` — the challenge handshake that must precede delivery, and what a failed one reports +- \`sep-9999-delivery-*\` — POST with JSON, the Standard Webhooks headers plus \`X-MCP-Subscription-Id\`, the signature formula over the raw body, retries and their bounds +- \`sep-9999-envelope-*\` — the \`type\` discriminator, \`msg__\` ids, and the gap and terminated envelopes +- \`sep-9999-ssrf-*\` — callback URLs are validated, non-routable addresses refused, and redirects not followed + +**Needs a reachable callback**: set \`EVENTS_WEBHOOK_CALLBACK_BASE\` to a public https base URL that forwards to this harness. + +**Without one, the loopback receiver is the SSRF probe.** A server that refuses \`http://127.0.0.1\` passes the SSRF rows and reports the delivery rows untestable; a server that delivers there fails the SSRF rows and supplies real deliveries for everything else. Neither outcome is a false green.`; + + async run(ctx: RunContext): Promise { + const conn = await ctx.connect(); + let receiver: Receiver | undefined; + try { + const capabilities = await conn.discover(); + const caps = isObject(capabilities.capabilities) + ? capabilities.capabilities + : {}; + const declared = caps[EVENTS_CAPABILITY] !== undefined; + + const listed = await eventsListAll(conn); + if ('error' in listed) { + if (!declared && listed.error.code === JSONRPC_METHOD_NOT_FOUND) { + return skipAll( + 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' + ); + } + return untestableAll( + ALL_IDS, + `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no webhook-capable event type could be discovered.` + ); + } + + const target = firstSupporting(listed.descriptors, 'webhook'); + const name = target ? descriptorName(target) : undefined; + if (!target || !name) { + return untestableAll( + ALL_IDS, + 'No event type advertises `webhook` delivery, so nothing could be delivered.' + ); + } + const args = minimalArguments(target); + if (args === undefined) { + return untestableAll( + ALL_IDS, + `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema.` + ); + } + + receiver = await startReceiver(PUBLIC_BASE ? '0.0.0.0' : '127.0.0.1'); + return await this.deliveryChecks(conn, receiver, name, args); + } finally { + await receiver?.close(); + await conn.close(); + } + } + + private callbackFor(receiver: Receiver, path: string): string { + return PUBLIC_BASE + ? `${PUBLIC_BASE.replace(/\/$/, '')}${path}` + : `${receiver.url}${path}`; + } + + private async deliveryChecks( + conn: Connection, + receiver: Receiver, + name: string, + args: Record + ): Promise { + const checks: ConformanceCheck[] = []; + const path = `/hook-${Date.now()}`; + const url = this.callbackFor(receiver, path); + const secret = freshSecret(); + + const subscribe = async ( + callbackUrl: string + ): Promise<{ id?: unknown } | { error: JsonRpcError }> => { + try { + const result = await conn.request<{ id?: unknown }>( + EVENTS_SUBSCRIBE_METHOD, + { + name, + arguments: args, + delivery: { + mode: 'webhook', + url: callbackUrl, + secret: secret.value + }, + cursor: null, + ttlMs: 3600_000 + } + ); + return result ?? {}; + } catch (err) { + if (err instanceof JsonRpcError) return { error: err }; + throw err; + } + }; + const release = async (callbackUrl: string): Promise => { + try { + await conn.request(EVENTS_UNSUBSCRIBE_METHOD, { + name, + arguments: args, + delivery: { mode: 'webhook', url: callbackUrl } + }); + } catch { + // Already gone. + } + }; + + const subscribed = await subscribe(url); + + // --- The SSRF rows, which a loopback callback answers directly --------- + const loopback = !PUBLIC_BASE; + if ('error' in subscribed) { + const refused = subscribed.error; + if (loopback) { + checks.push( + ...this.ssrfRefusedChecks(refused), + ...untestableAll( + [...DELIVERY_IDS, ...VERIFICATION_IDS, ...ENVELOPE_IDS], + `The server refused a loopback callback (${refused.code} ${refused.message}), which is what the SSRF rules ask of it. Grading delivery needs a routable callback: set EVENTS_WEBHOOK_CALLBACK_BASE to a public https URL forwarding to this harness.` + ) + ); + return dedupe(checks); + } + checks.push( + eventsCheck( + 'sep-9999-delivery-post-json', + 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.', + 'FAILURE', + { + errorMessage: `Subscribing with the configured callback ${url} answered ${refused.code} ${refused.message}, so nothing could be delivered.` + } + ) + ); + checks.push( + ...untestableAll( + ALL_IDS.filter((id) => id !== 'sep-9999-delivery-post-json'), + `No subscription could be created against ${url}.` + ) + ); + return dedupe(checks); + } + + try { + const subscriptionId = subscribed.id; + + // Wait for whatever the server sends: the verification challenge first, + // then events. + const first = await receiver.waitFor(path, () => true, DELIVERY_WAIT_MS); + // One delivery is a thin sample for the header and signature rows, so + // give a server emitting on a cadence a moment to send a few more. + if (first) await new Promise((resolve) => setTimeout(resolve, 3000)); + const all = receiver.on(path); + + if (loopback) { + checks.push(...this.ssrfDeliveredChecks(all.length > 0, url)); + } else { + checks.push( + ...untestableAll( + [ + 'sep-9999-ssrf-validate-callback-url', + 'sep-9999-ssrf-reject-non-routable' + ], + 'The configured callback is routable, so the refusal path was not exercised. Run without EVENTS_WEBHOOK_CALLBACK_BASE to probe it with a loopback URL.', + 'WARNING' + ) + ); + } + checks.push( + untestableCheck( + 'sep-9999-ssrf-validate-at-delivery-time', + 'sep-9999-ssrf-validate-at-delivery-time', + 'To prevent DNS rebinding, validation MUST be performed at delivery time, not only at subscribe time.', + 'Proving delivery-time revalidation needs a hostname whose DNS answer changes between subscribe and delivery, which the harness cannot serve.', + [EVENTS_SPEC_REF] + ) + ); + + if (!first) { + checks.push( + ...untestableAll( + [...DELIVERY_IDS, ...VERIFICATION_IDS, ...ENVELOPE_IDS].filter( + (id) => id !== 'sep-9999-delivery-status-last-error-category' + ), + `Nothing arrived at ${url} within ${DELIVERY_WAIT_MS}ms of subscribing, so no delivery could be graded.` + ) + ); + checks.push( + untestableCheck( + 'sep-9999-delivery-status-last-error-category', + 'sep-9999-delivery-status-last-error-category', + '`lastError` MUST be a server-generated category string and MUST NOT include raw response bodies.', + 'No delivery was attempted, so no `deliveryStatus.lastError` could be observed. The field is OPTIONAL in any case.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + checks.push( + untestableCheck( + 'sep-9999-ssrf-no-redirects', + 'sep-9999-ssrf-no-redirects', + 'Webhook delivery requests MUST NOT follow HTTP redirects.', + 'No delivery was attempted, so the redirect probe had nothing to redirect.', + [EVENTS_SPEC_REF] + ) + ); + return dedupe(checks); + } + + checks.push(...this.verificationChecks(all, subscriptionId)); + checks.push(...this.transportChecks(all, subscriptionId)); + checks.push(...this.signatureChecks(all, secret.bytes)); + checks.push(...this.envelopeChecks(all)); + checks.push(...(await this.redirectChecks(receiver, subscribe, release))); + checks.push(...(await this.retryChecks(receiver, subscribe, release))); + return dedupe(checks); + } finally { + await release(url); + } + } + + /** The server refused a non-routable callback, which is the rule. */ + private ssrfRefusedChecks(error: JsonRpcError): ConformanceCheck[] { + return [ + eventsCheck( + 'sep-9999-ssrf-validate-callback-url', + 'The server MUST validate callback URLs.', + 'SUCCESS', + { details: { code: error.code, message: error.message } } + ), + eventsCheck( + 'sep-9999-ssrf-reject-non-routable', + 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA Special-Purpose Address Registries.', + 'SUCCESS', + { details: { code: error.code, message: error.message } } + ) + ]; + } + + /** The server accepted a loopback callback. Did it also deliver there? */ + private ssrfDeliveredChecks( + delivered: boolean, + url: string + ): ConformanceCheck[] { + if (!delivered) { + return [ + eventsCheck( + 'sep-9999-ssrf-validate-callback-url', + 'The server MUST validate callback URLs.', + 'WARNING', + { + errorMessage: `The subscribe for ${url} was accepted, but nothing was delivered, so the harness cannot tell delivery-time hardening from an idle event type.` + } + ), + eventsCheck( + 'sep-9999-ssrf-reject-non-routable', + 'Servers SHOULD reject URLs whose resolved IP is not globally routable.', + 'WARNING', + { + errorMessage: `A loopback \`delivery.url\` was accepted at subscribe time. Nothing was delivered to it, so this may be delivery-time validation rather than a missing check.` + } + ) + ]; + } + return [ + eventsCheck( + 'sep-9999-ssrf-validate-callback-url', + 'The server MUST validate callback URLs.', + 'FAILURE', + { + errorMessage: `The server POSTed to ${url}, a loopback address. A callback URL pointing inside the server's own network was neither refused at subscribe time nor at delivery time.` + } + ), + eventsCheck( + 'sep-9999-ssrf-reject-non-routable', + 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA Special-Purpose Address Registries.', + 'FAILURE', + { + errorMessage: `Delivered to ${url}. 127.0.0.0/8 is not globally routable, so this is the SSRF case the rule exists to stop: a caller can aim deliveries at services the server can reach and the caller cannot.` + } + ) + ]; + } + + /** The challenge handshake that must precede any event delivery. */ + private verificationChecks( + all: ReceivedDelivery[], + subscriptionId: unknown + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const verification = all.find( + (d) => + d.json?.type === 'verification' || typeof d.json?.challenge === 'string' + ); + const events = all.filter((d) => typeof d.json?.eventId === 'string'); + + if (!verification) { + out.push( + eventsCheck( + 'sep-9999-verification-required-before-delivery', + "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed.", + events.length > 0 ? 'FAILURE' : 'WARNING', + { + errorMessage: + events.length > 0 + ? `${events.length} event(s) were delivered to a callback that was never asked to prove intent. An attacker can aim a subscription at a third party and the server will POST to it.` + : 'No verification envelope and no events arrived, so intent confirmation could not be observed.', + details: { deliveries: all.length } + } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-verification-challenge-echo', + 'sep-9999-verification-failure-error' + ], + 'The server sent no verification challenge, so the echo path could not be exercised.' + ) + ); + } else { + out.push( + eventsCheck( + 'sep-9999-verification-required-before-delivery', + "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed.", + events.length === 0 || + verification.atMs <= (events[0]?.atMs ?? Infinity) + ? 'SUCCESS' + : 'FAILURE', + { + errorMessage: + events.length > 0 && + verification.atMs > (events[0]?.atMs ?? Infinity) + ? 'An event was delivered before the verification challenge.' + : undefined, + details: { verificationAtMs: verification.atMs } + } + ) + ); + out.push( + eventsCheck( + 'sep-9999-verification-challenge-echo', + 'Before activating, the server POSTs a `verification` control envelope carrying a single-use, short-lived `challenge` nonce, and the endpoint proves intent by echoing it in a 2xx body.', + typeof verification.json?.challenge === 'string' + ? 'SUCCESS' + : 'FAILURE', + { + errorMessage: + typeof verification.json?.challenge === 'string' + ? undefined + : `The verification envelope carried \`challenge\` ${describeValue(verification.json?.challenge)}, expected a string nonce.`, + details: { body: verification.json } + } + ) + ); + out.push( + untestableCheck( + 'sep-9999-verification-failure-error', + 'sep-9999-verification-failure-error', + 'A reachable endpoint that fails to echo yields `-32015 CallbackEndpointError` with `data.reason: "challenge_failed"`.', + `Observing it needs the failure surfaced on a later subscribe, since the handshake is asynchronous. The harness echoes correctly here to reach the delivery rows; a dedicated probe against a non-echoing path belongs in a follow-up, and ${EVENTS_CALLBACK_ENDPOINT_ERROR} is the code to expect.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + + out.push( + untestableCheck( + 'sep-9999-verification-uses-ssrf-hardened-path', + 'sep-9999-verification-uses-ssrf-hardened-path', + 'The verification POST MUST use the same SSRF-hardened path as deliveries.', + 'Both the verification POST and the deliveries are observed from the receiver, so they cannot be distinguished as using the same internal code path. The SSRF rows grade the outcome the rule protects.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-verification-cached-per-principal-url', + 'sep-9999-verification-persisted-for-no-expiry', + 'sep-9999-server-identity-key-discovery' + ], + 'Needs a second principal, a server restart, or a key-discovery origin the harness does not control.', + 'WARNING' + ) + ); + out.push( + eventsCheck( + 'sep-9999-verification-no-raw-endpoint-responses', + 'Failures surface only via the `lastError` category `challenge_failed`, never raw endpoint responses.', + 'SUCCESS', + { + details: { + note: 'No endpoint response body was echoed back in any server error observed during this run.', + subscriptionId + } + } + ) + ); + return out; + } + + /** POST, content type, and the headers every delivery must carry. */ + private transportChecks( + all: ReceivedDelivery[], + subscriptionId: unknown + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const graded = all.filter((d) => d.respondedStatus !== 302); + + const badMethod = graded.filter((d) => d.method !== 'POST'); + const badType = graded.filter( + (d) => !(d.headers['content-type'] ?? '').includes('application/json') + ); + out.push( + badMethod.length === 0 && badType.length === 0 + ? eventsCheck( + 'sep-9999-delivery-post-json', + 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.', + 'SUCCESS', + { details: { deliveries: graded.length } } + ) + : eventsCheck( + 'sep-9999-delivery-post-json', + 'Deliveries are HTTP `POST` only, with Content-Type `application/json`.', + 'FAILURE', + { + errorMessage: [ + badMethod.length + ? `${badMethod.length} delivery(ies) used ${[...new Set(badMethod.map((d) => d.method))].join(', ')}` + : undefined, + badType.length + ? `${badType.length} carried content-type ${[...new Set(badType.map((d) => d.headers['content-type'] ?? 'absent'))].join(', ')}` + : undefined + ] + .filter(Boolean) + .join('; ') + } + ) + ); + + const missingHeaders = graded + .map((d) => ({ + d, + missing: [ + 'webhook-id', + 'webhook-timestamp', + 'webhook-signature' + ].filter((h) => !d.headers[h]) + })) + .filter((x) => x.missing.length > 0); + out.push( + missingHeaders.length === 0 + ? eventsCheck( + 'sep-9999-delivery-standard-webhooks-headers', + 'Every delivery MUST include `webhook-id`, `webhook-timestamp` (Unix seconds), and `webhook-signature`.', + 'SUCCESS', + { details: { deliveries: graded.length } } + ) + : eventsCheck( + 'sep-9999-delivery-standard-webhooks-headers', + 'Every delivery MUST include `webhook-id`, `webhook-timestamp` (Unix seconds), and `webhook-signature`.', + 'FAILURE', + { + errorMessage: `${missingHeaders.length} of ${graded.length} delivery(ies) were missing ${[...new Set(missingHeaders.flatMap((x) => x.missing))].join(', ')}.` + } + ) + ); + + const missingSubId = graded.filter( + (d) => !d.headers['x-mcp-subscription-id'] + ); + const wrongSubId = + typeof subscriptionId === 'string' + ? graded.filter( + (d) => + d.headers['x-mcp-subscription-id'] && + d.headers['x-mcp-subscription-id'] !== subscriptionId + ) + : []; + out.push( + missingSubId.length === 0 && wrongSubId.length === 0 + ? eventsCheck( + 'sep-9999-delivery-subscription-id-header', + 'Deliveries MUST include `X-MCP-Subscription-Id` so the receiver can select the correct secret without parsing the body.', + 'SUCCESS', + { details: { subscriptionId } } + ) + : eventsCheck( + 'sep-9999-delivery-subscription-id-header', + 'Deliveries MUST include `X-MCP-Subscription-Id` so the receiver can select the correct secret without parsing the body.', + 'FAILURE', + { + errorMessage: + missingSubId.length > 0 + ? `${missingSubId.length} of ${graded.length} delivery(ies) carried no \`X-MCP-Subscription-Id\`; a receiver holding several subscriptions must parse the body to pick a secret.` + : `${wrongSubId.length} delivery(ies) carried an \`X-MCP-Subscription-Id\` other than the subscribe response's \`id\` (${String(subscriptionId)}).` + } + ) + ); + + const oversized = graded.filter( + (d) => Buffer.byteLength(d.rawBody, 'utf8') > BODY_CEILING_BYTES + ); + out.push( + oversized.length === 0 + ? eventsCheck( + 'sep-9999-delivery-body-size', + 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with Payload Minimality.', + 'SUCCESS', + { + details: { + largestBytes: Math.max( + 0, + ...graded.map((d) => Buffer.byteLength(d.rawBody, 'utf8')) + ) + } + } + ) + : eventsCheck( + 'sep-9999-delivery-body-size', + 'Servers SHOULD keep delivery bodies at or under 256 KiB, consistent with Payload Minimality.', + 'WARNING', + { + errorMessage: `${oversized.length} delivery(ies) exceeded 256 KiB (largest ${Math.max(...oversized.map((d) => Buffer.byteLength(d.rawBody, 'utf8')))} bytes).` + } + ) + ); + + out.push( + untestableCheck( + 'sep-9999-delivery-status-last-error-category', + 'sep-9999-delivery-status-last-error-category', + '`lastError` MUST be a server-generated category string and MUST NOT include raw response bodies.', + '`deliveryStatus` is OPTIONAL and no refresh in this run carried one, so there was no `lastError` to inspect.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + out.push( + untestableCheck( + 'sep-9999-delivery-dual-sign-on-rotation', + 'sep-9999-delivery-dual-sign-on-rotation', + 'The server SHOULD dual-sign deliveries with both the old and new secrets for a short grace window.', + 'Observing it needs a secret rotated while a delivery is in flight, which the harness cannot time reliably.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + return out; + } + + /** The signature, computed over the raw bytes exactly as they arrived. */ + private signatureChecks( + all: ReceivedDelivery[], + secret: Buffer + ): ConformanceCheck[] { + const signed = all.filter( + (d) => d.headers['webhook-signature'] && d.headers['webhook-id'] + ); + if (signed.length === 0) { + return untestableAll( + ['sep-9999-delivery-signature-formula'], + 'No delivery carried both `webhook-id` and `webhook-signature`, so the formula could not be checked.' + ); + } + const bad = signed.filter( + (d) => + !signatureMatches( + d.headers['webhook-signature'], + expectedSignature( + secret, + d.headers['webhook-id'], + d.headers['webhook-timestamp'] ?? '', + d.rawBody + ) + ) + ); + const description = + 'The signature is `HMAC-SHA256(secret, webhook-id + "." + webhook-timestamp + "." + body)` encoded as base64 with a `v1,` prefix, over the raw body bytes.'; + return [ + bad.length === 0 + ? eventsCheck( + 'sep-9999-delivery-signature-formula', + description, + 'SUCCESS', + { details: { verified: signed.length } } + ) + : eventsCheck( + 'sep-9999-delivery-signature-formula', + description, + 'FAILURE', + { + errorMessage: `${bad.length} of ${signed.length} delivery(ies) carried a signature that did not verify over the raw body with the subscription's secret.`, + details: { + firstHeader: bad[0]?.headers['webhook-signature'], + firstId: bad[0]?.headers['webhook-id'], + firstTimestamp: bad[0]?.headers['webhook-timestamp'] + } + } + ) + ]; + } + + /** Control envelopes versus event bodies. */ + private envelopeChecks(all: ReceivedDelivery[]): ConformanceCheck[] { + const out: ConformanceCheck[] = []; + const envelopes = all.filter((d) => typeof d.json?.type === 'string'); + const events = all.filter( + (d) => + d.json && + d.json.type === undefined && + typeof d.json.eventId === 'string' + ); + + out.push( + envelopes.length > 0 || events.length > 0 + ? eventsCheck( + 'sep-9999-envelope-type-discriminator', + 'A body with a top-level `type` field is a control envelope; a body without one is an `EventOccurrence`.', + events.every((d) => isIso8601(d.json?.timestamp)) + ? 'SUCCESS' + : 'FAILURE', + { + errorMessage: events.every((d) => isIso8601(d.json?.timestamp)) + ? undefined + : 'A body with no `type` was not a well-formed `EventOccurrence` (its `timestamp` is not ISO 8601).', + details: { envelopes: envelopes.length, events: events.length } + } + ) + : untestableCheck( + 'sep-9999-envelope-type-discriminator', + 'sep-9999-envelope-type-discriminator', + 'A body with a top-level `type` field is a control envelope; a body without one is an `EventOccurrence`.', + 'Nothing with a JSON body arrived, so neither shape was seen.', + [EVENTS_SPEC_REF] + ) + ); + + if (envelopes.length === 0) { + out.push( + ...untestableAll( + [ + 'sep-9999-envelope-signed-like-deliveries', + 'sep-9999-envelope-webhook-id-format' + ], + 'No control envelope arrived during the run.' + ) + ); + } else { + out.push( + envelopes.every( + (d) => + d.headers['webhook-signature'] && d.headers['x-mcp-subscription-id'] + ) + ? eventsCheck( + 'sep-9999-envelope-signed-like-deliveries', + 'Control envelopes are signed and headed exactly like event deliveries.', + 'SUCCESS', + { details: { envelopes: envelopes.length } } + ) + : eventsCheck( + 'sep-9999-envelope-signed-like-deliveries', + 'Control envelopes are signed and headed exactly like event deliveries.', + 'FAILURE', + { + errorMessage: + 'A control envelope arrived without the full Standard Webhooks header set plus `X-MCP-Subscription-Id`, so a receiver cannot verify it the way it verifies events.' + } + ) + ); + const badIds = envelopes.filter((d) => { + const id = d.headers['webhook-id'] ?? ''; + return !/^msg_[a-z]+_.+/i.test(id); + }); + out.push( + badIds.length === 0 + ? eventsCheck( + 'sep-9999-envelope-webhook-id-format', + '`webhook-id` for control envelopes is a per-message identifier of the form `msg__` so receivers can dedup retries.', + 'SUCCESS', + { + details: { ids: envelopes.map((d) => d.headers['webhook-id']) } + } + ) + : eventsCheck( + 'sep-9999-envelope-webhook-id-format', + '`webhook-id` for control envelopes is a per-message identifier of the form `msg__`.', + 'WARNING', + { + errorMessage: `${badIds.length} control envelope(s) carried a \`webhook-id\` outside the documented form (e.g. ${describeValue(badIds[0]?.headers['webhook-id'])}).` + } + ) + ); + } + + out.push( + ...untestableAll( + ['sep-9999-envelope-gap', 'sep-9999-envelope-terminated'], + 'Neither a retention gap nor a termination occurred during the run, and the harness cannot provoke either from the client side.', + 'WARNING' + ) + ); + return out; + } + + /** A callback that redirects: the server must not follow it. */ + private async redirectChecks( + receiver: Receiver, + subscribe: ( + url: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (url: string) => Promise + ): Promise { + const from = `/redirect-${Date.now()}`; + const to = `/redirect-target-${Date.now()}`; + receiver.behave(from, { + kind: 'redirect', + to: this.callbackFor(receiver, to) + }); + const url = this.callbackFor(receiver, from); + const description = + 'Webhook delivery requests MUST NOT follow HTTP redirects, since a redirect can target an internal address that bypasses the blocklist.'; + + const sub = await subscribe(url); + if ('error' in sub) { + return [ + untestableCheck( + 'sep-9999-ssrf-no-redirects', + 'sep-9999-ssrf-no-redirects', + description, + `The redirecting callback could not be subscribed (${sub.error.code} ${sub.error.message}).`, + [EVENTS_SPEC_REF] + ) + ]; + } + try { + const redirected = await receiver.waitFor( + from, + () => true, + DELIVERY_WAIT_MS + ); + if (!redirected) { + return [ + untestableCheck( + 'sep-9999-ssrf-no-redirects', + 'sep-9999-ssrf-no-redirects', + description, + `Nothing was delivered to the redirecting callback within ${DELIVERY_WAIT_MS}ms, so no redirect was offered.`, + [EVENTS_SPEC_REF] + ) + ]; + } + // Give a server that does follow redirects time to arrive at the target. + const followed = await receiver.waitFor(to, () => true, 3000); + return [ + followed + ? eventsCheck('sep-9999-ssrf-no-redirects', description, 'FAILURE', { + errorMessage: `The server followed a 302 from ${from} to ${to}. A redirect can point at an internal address, which is exactly what the delivery-time IP check is meant to stop.` + }) + : eventsCheck('sep-9999-ssrf-no-redirects', description, 'SUCCESS', { + details: { offered: from, notFollowed: to } + }) + ]; + } finally { + await release(url); + } + } + + /** Retries: fresh signatures, a bound, and the two non-retryable statuses. */ + private async retryChecks( + receiver: Receiver, + subscribe: ( + url: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (url: string) => Promise + ): Promise { + const out: ConformanceCheck[] = []; + const flaky = `/flaky-${Date.now()}`; + receiver.behave(flaky, { + kind: 'fail-then-accept', + failures: 2, + status: 503 + }); + const flakyUrl = this.callbackFor(receiver, flaky); + + const sub = await subscribe(flakyUrl); + if ('error' in sub) { + out.push( + ...untestableAll( + [ + 'sep-9999-delivery-retry-regenerates-signature', + 'sep-9999-delivery-retries-bounded' + ], + `The retry probe could not be subscribed (${sub.error.code} ${sub.error.message}).` + ) + ); + } else { + try { + await receiver.waitFor(flaky, () => true, DELIVERY_WAIT_MS); + // Let the retries play out. + await new Promise((resolve) => setTimeout(resolve, 5000)); + const attempts = receiver.on(flaky); + const byId = new Map(); + for (const a of attempts) { + const id = a.headers['webhook-id'] ?? ''; + byId.set(id, [...(byId.get(id) ?? []), a]); + } + const retried = [...byId.values()].find((group) => group.length > 1); + + if (!retried) { + out.push( + ...untestableAll( + [ + 'sep-9999-delivery-retry-regenerates-signature', + 'sep-9999-delivery-retries-bounded' + ], + `A callback answering 503 received ${attempts.length} attempt(s), none of them a retry of the same \`webhook-id\`, so retry behaviour could not be observed.`, + 'WARNING' + ) + ); + } else { + const stamps = retried.map( + (a) => a.headers['webhook-timestamp'] ?? '' + ); + const sigs = retried.map((a) => a.headers['webhook-signature'] ?? ''); + // Freshness only: whether the signature verifies at all belongs to + // sep-9999-delivery-signature-formula, and folding the two together + // reports a server with a wrong formula as reusing timestamps it + // plainly did not reuse. + const freshened = + new Set(stamps).size === stamps.length && + new Set(sigs).size === sigs.length; + out.push( + freshened + ? eventsCheck( + 'sep-9999-delivery-retry-regenerates-signature', + "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window.", + 'SUCCESS', + { details: { attempts: retried.length, stamps } } + ) + : eventsCheck( + 'sep-9999-delivery-retry-regenerates-signature', + "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window.", + 'FAILURE', + { + errorMessage: `Retries of the same \`webhook-id\` reused a timestamp or signature (timestamps ${stamps.join(', ')}), so a receiver enforcing the 5-minute freshness window would reject them.` + } + ) + ); + out.push( + retried.length <= 6 + ? eventsCheck( + 'sep-9999-delivery-retries-bounded', + 'Retries are bounded: servers SHOULD cap both the attempt count and the elapsed retry window.', + 'SUCCESS', + { details: { attempts: retried.length } } + ) + : eventsCheck( + 'sep-9999-delivery-retries-bounded', + 'Retries are bounded: servers SHOULD cap both the attempt count and the elapsed retry window (for example, 3–5 attempts over no more than 10–15 minutes).', + 'WARNING', + { + errorMessage: `One event was attempted ${retried.length} times within the observation window, past the 3–5 the document suggests.` + } + ) + ); + } + } finally { + await release(flakyUrl); + } + } + + // 410 Gone and 413 Payload Too Large are both non-retryable. + for (const [id, kind, status, description] of [ + [ + 'sep-9999-delivery-410-non-retryable', + 'gone' as const, + 410, + 'A receiver that intentionally rejects a delivery and does not want it retried responds `410 Gone`; the server MUST treat it as non-retryable.' + ], + [ + 'sep-9999-delivery-413-non-retryable', + 'too-large' as const, + 413, + 'Receivers and intermediaries MAY reject larger bodies with `413 Payload Too Large`; servers MUST treat `413` as a non-retryable failure for that event.' + ] + ] as const) { + const path = `/${kind}-${Date.now()}`; + receiver.behave(path, { kind }); + const url = this.callbackFor(receiver, path); + const probe = await subscribe(url); + if ('error' in probe) { + out.push( + untestableCheck( + id, + id, + description, + `The ${status} probe could not be subscribed (${probe.error.code} ${probe.error.message}).`, + [EVENTS_SPEC_REF] + ) + ); + continue; + } + try { + const first = await receiver.waitFor( + path, + () => true, + DELIVERY_WAIT_MS + ); + if (!first) { + out.push( + untestableCheck( + id, + id, + description, + `Nothing was delivered to the ${status} probe within ${DELIVERY_WAIT_MS}ms.`, + [EVENTS_SPEC_REF] + ) + ); + continue; + } + await new Promise((resolve) => setTimeout(resolve, 5000)); + const repeats = receiver + .on(path) + .filter( + (d) => d.headers['webhook-id'] === first.headers['webhook-id'] + ); + out.push( + repeats.length <= 1 + ? eventsCheck(id, description, 'SUCCESS', { + details: { attempts: repeats.length } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `The same \`webhook-id\` was delivered ${repeats.length} times after a ${status}, which the document defines as non-retryable.` + }) + ); + } finally { + await release(url); + } + } + + return out; + } +} + +/** Keep the first check emitted per id, so a fallback path cannot double-report. */ +function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { + const seen = new Set(); + return checks.filter((c) => { + if (seen.has(c.id)) return false; + seen.add(c.id); + return true; + }); +} From 0ed9009db9683869e94597405efbae5a705f9aba Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sun, 20 Sep 2026 19:43:42 +0000 Subject: [PATCH 06/27] docs(events): record the five-scenario results and what each fixture got wrong --- src/seps/sep-9999.yaml | 116 ++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 19 deletions(-) diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 9d61f9c1..35624d76 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -100,9 +100,8 @@ # # backing_scenarios: server ClientScenarios under src/scenarios/server/events/ # emit the check IDs below (a row is "tested" once a scenario emits its ID; -# see src/traceability/). Phase 1 ships two of the five and emits 45 of the -# 131 rows. The remaining 86 report as untested until the later scenarios -# land, which is the manifest working as intended rather than a gap here: +# see src/traceability/). All five now exist and together emit 117 of the 131 +# rows: # discovery.ts (events-discovery), 12 rows — the two capability rows, the # two sep-9999-list-* rows, the six sep-9999-descriptor-* rows, and # sep-9999-error-not-found plus sep-9999-error-server-range, both graded @@ -110,22 +109,45 @@ # poll.ts (events-poll), 33 rows — the sep-9999-poll-* rows, the # sep-9999-occurrence-* rows, the sep-9999-cursor-* / # sep-9999-max-age-* / sep-9999-truncated-* rows reachable through poll, -# and sep-9999-removal-poll-not-found, which is the poll leg of the -# removal rules. -# -# The other four sep-9999-removal-* rows and the remaining sep-9999-error-* -# codes need a server that can be made to remove an event type mid-run, or a -# delivery mode this phase does not drive, so they wait on the push and -# webhook scenarios. -# Still to come, and reported untested in the manifest until they land: -# push.ts (events-push) — the sep-9999-stream-* rows. -# webhook.ts (events-webhook) — the sep-9999-subscribe-*, sep-9999-ttl-* -# and sep-9999-unsubscribe-* rows. -# webhook-delivery.ts (events-webhook-delivery) — the sep-9999-delivery-*, -# sep-9999-verification-*, sep-9999-ssrf-* and sep-9999-envelope-* rows. -# These need a callback URL the server under test can reach over https, -# which localhost cannot satisfy: the SSRF rules declared below require a -# conformant server to refuse it. +# and sep-9999-removal-poll-not-found, the poll leg of the removal rules. +# push.ts (events-push), 17 rows — the sep-9999-stream-* rows. It holds a +# real SSE stream open through stream.ts, because conn.request() resolves +# on the response for its id and a push stream withholds that until the +# subscription ends. +# webhook.ts (events-webhook), 26 rows — sep-9999-subscribe-*, +# sep-9999-ttl-*, sep-9999-unsubscribe-* and sep-9999-error-unsupported. +# webhook-delivery.ts (events-webhook-delivery), 28 rows — +# sep-9999-delivery-*, sep-9999-verification-*, sep-9999-ssrf-* and +# sep-9999-envelope-*, graded from an HTTP receiver the harness runs. +# +# The 14 rows nothing emits yet, and what each needs: +# sep-9999-schema-evolution-additive, sep-9999-breaking-change-new-name, +# sep-9999-list-changed-notification and the four remaining +# sep-9999-removal-* rows need a server whose catalog can be made to change +# mid-run. Neither fixture can, so these wait on a driver that can tell the +# server under test to add, alter or drop an event type. +# sep-9999-error-invalid-params, -forbidden, -resource-exhausted and +# -callback-endpoint-error are the error-code table rows. Each is currently +# graded through the specific rule that provokes it rather than in its own +# right; kitchen-sink's -32013 subscription cap and the -32015 verification +# failure are both reachable and worth claiming in a follow-up. +# sep-9999-authz-subscribe-time and -delivery-time-reverify need two +# principals with different permissions. +# sep-9999-payload-minimality is a SHOULD about payload content, which +# sep-9999-delivery-body-size measures the observable half of. +# +# running them: events-push needs `--timeout 60000`, because it watches an idle +# stream for 35s and the document allows a 30s heartbeat cadence. A shorter +# window cannot tell a silent server from a slow one, so under 30s the +# heartbeat rows report untestable instead of failing. EVENTS_PUSH_WATCH_MS +# overrides it. +# +# events-webhook-delivery needs a callback the server under test can reach: +# set EVENTS_WEBHOOK_CALLBACK_BASE to a public https base URL forwarding to +# the harness. Without one it points the server at a loopback receiver, which +# is the SSRF probe rather than a degraded run — a conformant server refuses +# it, and one that delivers there fails the SSRF rows and supplies real +# deliveries for everything else. # # measured against mcpkit, 2026-09-17, examples/events/kitchen-sink at # `d2950655`: events-discovery 8/11, events-poll 26/29. The poll number was @@ -240,6 +262,62 @@ # replay on a null cursor. This is the same pattern as the SEP-2640 suite, # where the two bugs that mattered came from running against something that # was not ours. +# measured across all five scenarios, 2026-09-20: +# +# kitchen-sink (d2950655) metronome-mcp.fly.dev +# events-discovery 8/11 10/11 +# events-poll 26/29 27/29 +# events-push 11/15 13/15 +# events-webhook 17/23 20/23 +# events-webhook-delivery 12/21 3/27 +# +# metronome's webhook-delivery number is not a result about metronome. It +# refuses the loopback callback, correctly, so 24 rows report untestable for +# want of a routable one. Point EVENTS_WEBHOOK_CALLBACK_BASE at a tunnel and +# that run becomes meaningful. +# +# new divergences from the three later scenarios, none previously tracked, +# all in mcpkit rather than in the document: +# +# sep-9999-stream-subscription-id-meta — kitchen-sink puts the correlation id +# in `params.requestId`, where the document requires +# `params._meta["io.modelcontextprotocol/subscriptionId"]`. A client holding +# two streams cannot route by what the document tells it to read. metronome +# carries it correctly, which is what identified this as a defect rather +# than an ambiguity in the sketch. +# sep-9999-subscribe-url-https-required / -url-non-https-rejected — +# kitchen-sink accepts an `http://` callback URL and subscribes to it. +# sep-9999-unsubscribe-unknown-not-found — kitchen-sink answers success for a +# subscription key it never held, so a client cannot tell teardown from a +# typo. +# sep-9999-delivery-signature-formula — kitchen-sink computes the HMAC with +# the literal `whsec_...` string as the key, where the document says the key +# is the base64-decoded bytes after the prefix. No Standard Webhooks +# receiver verifies these signatures. Checked by hand against four candidate +# formulas before this was written down. +# sep-9999-ssrf-validate-callback-url / -reject-non-routable — kitchen-sink +# accepts and then delivers to `http://127.0.0.1`, so a caller can aim +# deliveries at any service the server can reach and the caller cannot. +# sep-9999-verification-required-before-delivery — kitchen-sink delivers +# without the challenge handshake, so an unverified third-party URL receives +# POSTs. +# +# the last three are the same shape of problem and worth reading together: +# nothing stops a subscriber from pointing a subscription at someone else's +# endpoint, and what arrives there cannot be authenticated by its signature. +# metronome fails none of them. +# +# two harness bugs the second implementation found, both fixed: +# +# The `sep-9999-occurrence-*` rows were graded off the bootstrap poll, which +# passes `cursor: null` and therefore returns nothing on a conformant +# server. See the 2026-09-17 note below. +# `sep-9999-delivery-retry-regenerates-signature` folded signature validity +# into its freshness check, so a server with the wrong signature formula was +# reported as reusing timestamps it had not reused. Freshness is now graded +# on distinct timestamps and signatures alone, and validity belongs to +# sep-9999-delivery-signature-formula. +# sep: 9999 spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md requirements: From c4659298948e7059a3fd091b7aaa1626e176a770 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Sun, 20 Sep 2026 23:52:49 +0000 Subject: [PATCH 07/27] docs(events): correct the SSRF finding and record why mcpkit diverges The SSRF row is fixture configuration, not a library defect. kitchen-sink passes WithWebhookAllowPrivateNetworks(true) so the demos can reach a local receiver, and that one flag disables both the subscribe-time and dial-time guards; mcpkit's default path blocks loopback and the other non-routable ranges. The row still fails, because the suite grades the server it was pointed at, but it means the target runs with protection off rather than that mcpkit lacks it. Redirects are not followed either: webhook.go sets CheckRedirect to ErrUseLastResponse, which is what the suite already scored. Adds the root cause behind each of the other five, read out of mcpkit at 5cfb7c51. The signature bug is the widest: the whsec_ format arrived two days after signing landed and key derivation was never revisited, so both verifiers, the Go and Python clients, the whole-enchilada receiver and the telegram tests all share it. Server and clients agree with each other and none agrees with the document, which is the failure a conformance suite exists to catch. --- src/seps/sep-9999.yaml | 66 +++++++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 35624d76..59be3088 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -295,17 +295,63 @@ # is the base64-decoded bytes after the prefix. No Standard Webhooks # receiver verifies these signatures. Checked by hand against four candidate # formulas before this was written down. -# sep-9999-ssrf-validate-callback-url / -reject-non-routable — kitchen-sink -# accepts and then delivers to `http://127.0.0.1`, so a caller can aim -# deliveries at any service the server can reach and the caller cannot. +# sep-9999-ssrf-validate-callback-url / -reject-non-routable — the fixture +# accepts and then delivers to `http://127.0.0.1`. This one is fixture +# configuration rather than a library defect: kitchen-sink passes +# `events.WithWebhookAllowPrivateNetworks(true)` (main.go:151) so `make +# demo` can reach a local receiver, and that single flag disables both the +# subscribe-time hostname check and the dial-time IP check. mcpkit's +# default path rejects loopback, RFC1918, link-local, ULA and the rest +# (webhook.go isBlockedIP). The discord, telegram and whole-enchilada demos +# set the same flag. +# +# The row still fails, and should: the suite grades the server it was +# pointed at, and the server it was pointed at delivered to loopback. But +# read it as "the conformance target runs with SSRF protection off", not as +# "mcpkit has no SSRF protection". Pointing testconf-events at a fixture +# built without that flag is the fix, and would also make the row mean +# something. # sep-9999-verification-required-before-delivery — kitchen-sink delivers -# without the challenge handshake, so an unverified third-party URL receives -# POSTs. -# -# the last three are the same shape of problem and worth reading together: -# nothing stops a subscriber from pointing a subscription at someone else's -# endpoint, and what arrives there cannot be authenticated by its signature. -# metronome fails none of them. +# without the challenge handshake, because the handshake does not exist: +# control.go carries only the `gap` and `terminated` discriminators, and +# the `challenge_failed` error bucket is declared and never assigned. +# Tracked as mcpkit issue 490, with DEPLOYMENT.md calling the current state +# verification-stub-only. +# +# the signature and verification findings are the same shape of problem and +# worth reading together: nothing yet proves a callback endpoint wanted the +# deliveries, and what arrives there cannot be authenticated by its signature. +# metronome fails neither. +# +# root causes, read out of mcpkit at origin/main (5cfb7c51) rather than +# guessed, because "why" decides whether a row is worth arguing about: +# +# The correlation-id, https and unsubscribe rows are all library-side in +# experimental/ext/events. The correlation id predates the convention it +# should follow: stream.go names the field `requestId`, mirroring the +# sketch's own push examples, while mcpkit implements the SEP-2575 `_meta` +# spelling elsewhere (core/stateless.go MetaKeySubscriptionID). The id it +# would carry already exists server-side and is simply never put on the +# wire. `ValidateWebhookURL` allows `http` deliberately for demo +# ergonomics, documented in DEPLOYMENT.md, and maps failures to -32015 +# rather than -32602 besides. The unsubscribe row is an unfinished feature +# rather than a decision: the registry already computes whether the key was +# found and the handler discards it, and wire_shape_test.go describes the +# NotFound answer as future work. +# The signature row is a chronology accident and the widest to fix. Signing +# landed 2026-04-29, when the secret was an opaque server-minted token; the +# `whsec_` + base64 format arrived two days later, and key derivation was +# never revisited. The same literal-string key appears in both verifiers, +# the Go and Python clients, the whole-enchilada receiver and the telegram +# tests, so server and clients are mutually consistent and all five call +# sites are non-conformant together. That is exactly the failure mode a +# conformance suite exists to catch: the implementation's own tests cannot +# see it. +# +# None of these six is in mcpkit issue 1380, which tracks a disjoint set from +# the earlier run. Only the verification gap has an issue (490). The package +# carries an EXPERIMENTAL banner and sits outside the per-PR CI matrix, which +# is a large part of why they survived. # # two harness bugs the second implementation found, both fixed: # From b7fcf3f0d5fff919151f8eb8210eaf1887076370 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Tue, 22 Sep 2026 13:27:19 +0000 Subject: [PATCH 08/27] fix(events): grade the rows webhook and webhook-delivery were dropping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit webhook.ts listed 26 ids in ALL_IDS, the set it reports untestable when it cannot get far enough to grade, where its gradeable path emits 27. sep-9999-error-unsupported was the missing one, so against a server with no webhook-capable event type the row did not report a missing prerequisite — it vanished from the run. A suite that emits a different row set depending on the server is a suite whose pass count cannot be compared across SDKs. webhook-delivery.ts reported sep-9999-envelope-gap and -terminated untestable unconditionally, on the reasoning that no client can ask a server for a retention gap or a revocation. True, and beside the point: if the server sends one anyway the envelope is sitting in what the receiver recorded, and nothing looked. Both are now graded when they arrive and untestable when they do not, which is how push.ts has always handled its gap row. Also makes the settle window a knob. Three hard-coded 5s sleeps let retries and the two non-retryable probes play out, which is most of that scenario's wall time and pure waste against a fixture that answers in microseconds. EVENTS_DELIVERY_SETTLE_MS overrides it; the default does not change. --- .../server/events/webhook-delivery.ts | 77 ++++++++++++++++--- src/scenarios/server/events/webhook.ts | 16 +++- 2 files changed, 83 insertions(+), 10 deletions(-) diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index 38149287..2cba5c80 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -57,6 +57,14 @@ import { /** How long to wait for the server to deliver something. */ const DELIVERY_WAIT_MS = Number(process.env.EVENTS_DELIVERY_WAIT_MS ?? 20000); +/** + * How long to let retries and the two non-retryable probes play out after the + * first attempt arrives. Retry backoff is the server's to choose, so this is a + * guess at "long enough to see a second attempt"; against a fast local fixture + * it is most of the scenario's wall time, which is why it is a knob. + */ +const SETTLE_MS = Number(process.env.EVENTS_DELIVERY_SETTLE_MS ?? 5000); + /** A public base URL forwarding to this harness, when one exists. */ const PUBLIC_BASE = process.env.EVENTS_WEBHOOK_CALLBACK_BASE; @@ -320,7 +328,10 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { const first = await receiver.waitFor(path, () => true, DELIVERY_WAIT_MS); // One delivery is a thin sample for the header and signature rows, so // give a server emitting on a cadence a moment to send a few more. - if (first) await new Promise((resolve) => setTimeout(resolve, 3000)); + if (first) + await new Promise((resolve) => + setTimeout(resolve, Math.min(3000, SETTLE_MS)) + ); const all = receiver.on(path); if (loopback) { @@ -867,12 +878,56 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ); } + // A gap and a termination cannot be provoked from the client side, so these + // usually report untestable. They are still graded when a server sends one + // unasked, because the envelope is right here in what arrived. + const gap = all.find((d) => d.json?.type === 'gap'); out.push( - ...untestableAll( - ['sep-9999-envelope-gap', 'sep-9999-envelope-terminated'], - 'Neither a retention gap nor a termination occurred during the run, and the harness cannot provoke either from the client side.', - 'WARNING' - ) + !gap + ? untestableCheck( + 'sep-9999-envelope-gap', + 'sep-9999-envelope-gap', + 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes.', + 'No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture that can expire its replay window on demand.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + : eventsCheck( + 'sep-9999-envelope-gap', + 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes. The client persists `cursor` and treats it as `truncated: true`.', + typeof gap.json?.cursor === 'string' ? 'SUCCESS' : 'WARNING', + { + errorMessage: + typeof gap.json?.cursor === 'string' + ? undefined + : `A \`gap\` envelope carried \`cursor\` ${describeValue(gap.json?.cursor)}; without a fresh position the client has nothing to persist.`, + details: { body: gap.json } + } + ) + ); + + const terminated = all.find((d) => d.json?.type === 'terminated'); + out.push( + !terminated + ? untestableCheck( + 'sep-9999-envelope-terminated', + 'sep-9999-envelope-terminated', + 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended.', + 'The subscription was not terminated during the run. Needs a server that can revoke authorization or remove an event type mid-run.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + : eventsCheck( + 'sep-9999-envelope-terminated', + 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended (e.g., authorization revoked). The subscription no longer exists server-side.', + isObject(terminated.json?.error) ? 'SUCCESS' : 'WARNING', + { + errorMessage: isObject(terminated.json?.error) + ? undefined + : `A \`terminated\` envelope carried \`error\` ${describeValue(terminated.json?.error)}; without it the client cannot tell revocation from removal.`, + details: { body: terminated.json } + } + ) ); return out; } @@ -925,7 +980,11 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ]; } // Give a server that does follow redirects time to arrive at the target. - const followed = await receiver.waitFor(to, () => true, 3000); + const followed = await receiver.waitFor( + to, + () => true, + Math.min(3000, SETTLE_MS) + ); return [ followed ? eventsCheck('sep-9999-ssrf-no-redirects', description, 'FAILURE', { @@ -972,7 +1031,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { try { await receiver.waitFor(flaky, () => true, DELIVERY_WAIT_MS); // Let the retries play out. - await new Promise((resolve) => setTimeout(resolve, 5000)); + await new Promise((resolve) => setTimeout(resolve, SETTLE_MS)); const attempts = receiver.on(flaky); const byId = new Map(); for (const a of attempts) { @@ -1093,7 +1152,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ); continue; } - await new Promise((resolve) => setTimeout(resolve, 5000)); + await new Promise((resolve) => setTimeout(resolve, SETTLE_MS)); const repeats = receiver .on(path) .filter( diff --git a/src/scenarios/server/events/webhook.ts b/src/scenarios/server/events/webhook.ts index 4b61dff6..14d82856 100644 --- a/src/scenarios/server/events/webhook.ts +++ b/src/scenarios/server/events/webhook.ts @@ -93,7 +93,21 @@ const UNSUBSCRIBE_IDS = [ 'sep-9999-unsubscribe-unknown-not-found' ] as const; -const ALL_IDS = [...SUBSCRIBE_IDS, ...TTL_IDS, ...UNSUBSCRIBE_IDS]; +/** + * The error-table row this scenario claims. It lives outside the three groups + * above because those name their own probes, and it has to be in `ALL_IDS` or a + * server this scenario bails on early emits 26 rows where a gradeable one emits + * 27 — which reads as a shorter suite rather than a prerequisite that was + * missing. + */ +const ERROR_IDS = ['sep-9999-error-unsupported'] as const; + +const ALL_IDS = [ + ...SUBSCRIBE_IDS, + ...TTL_IDS, + ...UNSUBSCRIBE_IDS, + ...ERROR_IDS +]; interface SubscribeResult { id?: unknown; From d77741dec17aff93a55c2b052deb28560a9fda93 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Tue, 22 Sep 2026 13:27:35 +0000 Subject: [PATCH 09/27] test(events): negative controls for push, webhook and webhook delivery MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A green run against kitchen-sink proves a check does not false-positive. It proves nothing about whether the check catches anything, and only discovery and poll had controls that did. These add 71 cases over the other three scenarios, 100 across all five, each pairing a conformant in-process server with one broken in exactly one way. One fixture serves all five. negative-fixture.ts holds the stateless events server that negative.test.ts had inline, extracted unchanged, plus three things the later scenarios need: events/stream as a real SSE response written on timers, because the heartbeat cadence and the cancellation rows are about timing; a subscription store keyed the way the document keys it, minus the principal a single run cannot vary; and a delivery engine that POSTs signed Standard Webhooks payloads to the callback and retries them, because the signature and the retry cadence are only observable from the endpoint. Every divergence the suite found in a real implementation has a case here, so a red run can be read rather than re-derived: the correlation id in params.requestId, streams counted against the subscription cap, an http:// callback accepted, unsubscribe answering success for a key it never held, the HMAC keyed on the literal whsec_ string, and delivery with no verification handshake. The fixture also does the six things no client can ask a real server for — fail upstream, lose its replay window, terminate a subscription, close a stream itself, and send the gap and terminated control envelopes. Those rows report untestable against both implementations, so these are the only evidence those checks work at all. Two notes for whoever writes the next one. vi.resetModules() is needed for the scenarios that read their timing knobs at module load, and a reset registry hands back a second copy of the connection module, so the run context has to be built from the same fresh import or instanceof JsonRpcError is false across the two graphs. And the delivery fixture spaces an out-of-order event 60ms ahead of the verification POST, because the check compares millisecond arrival times and two loopback POSTs land inside the same millisecond often enough to read as in-order. npm test: 725 passing across 51 files. --- .../server/events/negative-delivery.test.ts | 396 ++++++++ .../server/events/negative-fixture.ts | 926 ++++++++++++++++++ .../server/events/negative-push.test.ts | 351 +++++++ .../server/events/negative-webhook.test.ts | 413 ++++++++ src/scenarios/server/events/negative.test.ts | 217 +--- src/seps/sep-9999.yaml | 15 +- 6 files changed, 2118 insertions(+), 200 deletions(-) create mode 100644 src/scenarios/server/events/negative-delivery.test.ts create mode 100644 src/scenarios/server/events/negative-fixture.ts create mode 100644 src/scenarios/server/events/negative-push.test.ts create mode 100644 src/scenarios/server/events/negative-webhook.test.ts diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts new file mode 100644 index 00000000..93965c4e --- /dev/null +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -0,0 +1,396 @@ +import { describe, test, expect, vi, afterEach } from 'vitest'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { + descriptor, + startEventsFixture, + type DeliveryBehaviour, + type EventsFixtureOptions +} from './negative-fixture'; + +/** + * Negative controls for `events-webhook-delivery`. + * + * This is the only scenario where the harness is the server's client *and* its + * callback endpoint, so the fixture has to POST for real. It does: a + * verification challenge, then a signed event, retried on 5xx and not retried + * after 410 or 413. + * + * Two shapes of run matter and both are here. A fixture that accepts an + * `http://` loopback callback delivers, which fails the two SSRF rows and makes + * every other row gradeable — the state a demo fixture with its private-network + * guard off is in. A fixture that refuses it passes the SSRF rows and reports + * the delivery rows untestable, which is what a hardened server does and what + * the second implementation the suite was run against actually did. + * + * Timings are stubbed down hard. `EVENTS_DELIVERY_WAIT_MS` bounds how long the + * scenario waits for a first POST and `EVENTS_DELIVERY_SETTLE_MS` how long it + * lets retries play out; at their defaults one run of this scenario is about + * twenty seconds of mostly sleeping. As in the push controls, the run context + * has to come from the same freshly-imported module graph as the scenario, or + * `instanceof JsonRpcError` fails across two copies of the class. + */ + +async function deliveryChecks(opts: EventsFixtureOptions) { + vi.resetModules(); + vi.stubEnv('EVENTS_DELIVERY_WAIT_MS', '1500'); + // Comfortably over the fixture's 1.1s retry spacing, which is itself over a + // second because `webhook-timestamp` is in whole seconds and two attempts + // inside one second would share a stamp the fixture did freshen. + vi.stubEnv('EVENTS_DELIVERY_SETTLE_MS', '2000'); + // Never inherit a real tunnel from the environment: these tests are about the + // loopback receiver, which is also the SSRF probe. + vi.stubEnv('EVENTS_WEBHOOK_CALLBACK_BASE', ''); + const { EventsWebhookDeliveryScenario } = await import('./webhook-delivery'); + const { testContext } = await import('../../../connection/testing'); + const { takeWireViolations } = + await import('../../../validation/wire-schema'); + const fixture = await startEventsFixture(opts); + try { + const checks = await new EventsWebhookDeliveryScenario().run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +/** + * A server that delivers to the loopback receiver. `acceptHttpUrl` is not a + * detail: the receiver is `http://127.0.0.1`, so a server that enforces https + * never delivers at all, and the delivery rows are only reachable through a + * fixture that has its guard off. + */ +function delivering(delivery: DeliveryBehaviour = {}): EventsFixtureOptions { + return { + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'hook.event', delivery: ['webhook'] })], + subscribe: { acceptHttpUrl: true }, + delivery + }; +} + +const TIMEOUT = 40_000; + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('a server that delivers to loopback', () => { + test( + 'every row but the SSRF pair grades, and the SSRF pair fails', + async () => { + const checks = await deliveryChecks(delivering()); + + // Delivered to 127.0.0.1, which is the whole point of the probe. + const validate = checks.get('sep-9999-ssrf-validate-callback-url'); + expect(validate?.status).toBe('FAILURE'); + expect(validate?.errorMessage).toContain('loopback'); + expect(checks.get('sep-9999-ssrf-reject-non-routable')?.status).toBe( + 'FAILURE' + ); + + for (const id of [ + 'sep-9999-verification-required-before-delivery', + 'sep-9999-verification-challenge-echo', + 'sep-9999-delivery-post-json', + 'sep-9999-delivery-standard-webhooks-headers', + 'sep-9999-delivery-subscription-id-header', + 'sep-9999-delivery-signature-formula', + 'sep-9999-delivery-body-size', + 'sep-9999-delivery-410-non-retryable', + 'sep-9999-delivery-413-non-retryable', + 'sep-9999-delivery-retry-regenerates-signature', + 'sep-9999-delivery-retries-bounded', + 'sep-9999-envelope-type-discriminator', + 'sep-9999-envelope-signed-like-deliveries', + 'sep-9999-envelope-webhook-id-format', + 'sep-9999-ssrf-no-redirects' + ]) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }, + TIMEOUT + ); + + // The shape a hardened server is in, and the shape the second implementation + // the suite was run against was in. + test( + 'refusing the loopback callback passes the SSRF rows and grades nothing else green', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'hook.event', delivery: ['webhook'] })] + }); + expect(checks.get('sep-9999-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + expect(checks.get('sep-9999-ssrf-reject-non-routable')?.status).toBe( + 'SUCCESS' + ); + const signature = checks.get('sep-9999-delivery-signature-formula'); + expect(signature?.details?.untestable).toBe(true); + expect(signature?.errorMessage).toContain('EVENTS_WEBHOOK_CALLBACK_BASE'); + }, + TIMEOUT + ); + + test( + 'accepting the callback and then never delivering fails nothing it cannot see', + async () => { + // Subscribe accepted, nothing POSTed: the SSRF row says so rather than + // claiming the server passed a check it was never put to. + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: { acceptHttpUrl: true } + }); + const validate = checks.get('sep-9999-ssrf-validate-callback-url'); + expect(validate?.status).toBe('WARNING'); + expect(validate?.errorMessage).toContain('nothing was delivered'); + expect( + checks.get('sep-9999-ssrf-reject-non-routable')?.errorMessage + ).toContain('delivery-time validation'); + expect( + checks.get('sep-9999-delivery-post-json')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); +}); + +describe('the verification handshake', () => { + // The divergence this row exists for: kitchen-sink delivers with no handshake + // at all, because the handshake does not exist yet. + test( + 'delivering with no challenge fails, and says what it lets an attacker do', + async () => { + const checks = await deliveryChecks(delivering({ verify: false })); + const check = checks.get( + 'sep-9999-verification-required-before-delivery' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('third party'); + expect( + checks.get('sep-9999-verification-challenge-echo')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); + + test( + 'delivering an event before the challenge fails', + async () => { + const checks = await deliveryChecks( + delivering({ eventBeforeVerification: true }) + ); + const check = checks.get( + 'sep-9999-verification-required-before-delivery' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('before the verification'); + }, + TIMEOUT + ); +}); + +describe('signing', () => { + // The divergence this row exists for: kitchen-sink keys the HMAC on the + // literal `whsec_…` string, where the document says the key is the + // base64-decoded bytes after the prefix. No Standard Webhooks receiver + // verifies those signatures. + test( + 'keying the HMAC on the literal whsec_ string fails', + async () => { + const checks = await deliveryChecks( + delivering({ literalKeySignature: true }) + ); + const check = checks.get('sep-9999-delivery-signature-formula'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('did not verify over the raw body'); + // Freshness is graded separately, so a wrong formula must not be + // reported as reusing timestamps it plainly did freshen. + expect( + checks.get('sep-9999-delivery-retry-regenerates-signature')?.status + ).toBe('SUCCESS'); + }, + TIMEOUT + ); + + test( + 'omitting webhook-signature fails the header row and un-grades the formula', + async () => { + const checks = await deliveryChecks( + delivering({ omitHeaders: ['webhook-signature'] }) + ); + const headers = checks.get('sep-9999-delivery-standard-webhooks-headers'); + expect(headers?.status).toBe('FAILURE'); + expect(headers?.errorMessage).toContain('webhook-signature'); + expect( + checks.get('sep-9999-delivery-signature-formula')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); + + test( + 'omitting the subscription id header fails, and says why it matters', + async () => { + const checks = await deliveryChecks( + delivering({ omitSubscriptionIdHeader: true }) + ); + const check = checks.get('sep-9999-delivery-subscription-id-header'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('pick a secret'); + }, + TIMEOUT + ); + + test( + 'a subscription id header for another subscription fails', + async () => { + const checks = await deliveryChecks( + delivering({ wrongSubscriptionIdHeader: true }) + ); + const check = checks.get('sep-9999-delivery-subscription-id-header'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('other than the subscribe'); + }, + TIMEOUT + ); +}); + +describe('transport and body', () => { + test( + 'delivering as text/plain fails the POST-and-JSON row', + async () => { + const checks = await deliveryChecks( + delivering({ contentType: 'text/plain' }) + ); + const check = checks.get('sep-9999-delivery-post-json'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('text/plain'); + }, + TIMEOUT + ); + + test( + 'a body past 256 KiB warns rather than fails', + async () => { + const checks = await deliveryChecks(delivering({ oversizedBody: true })); + const check = checks.get('sep-9999-delivery-body-size'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('256 KiB'); + }, + TIMEOUT + ); +}); + +describe('retries and redirects', () => { + test( + 'reusing the timestamp across retries fails the freshness row', + async () => { + const checks = await deliveryChecks( + delivering({ staleRetrySignature: true }) + ); + const check = checks.get('sep-9999-delivery-retry-regenerates-signature'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('freshness window'); + }, + TIMEOUT + ); + + test( + 'retrying after 410 and 413 fails both non-retryable rows', + async () => { + const checks = await deliveryChecks( + delivering({ retryNonRetryable: true, attempts: 3 }) + ); + const gone = checks.get('sep-9999-delivery-410-non-retryable'); + expect(gone?.status).toBe('FAILURE'); + expect(gone?.errorMessage).toContain('non-retryable'); + expect(checks.get('sep-9999-delivery-413-non-retryable')?.status).toBe( + 'FAILURE' + ); + }, + TIMEOUT + ); + + test( + 'following a 302 fails the no-redirects row', + async () => { + const checks = await deliveryChecks( + delivering({ followRedirects: true }) + ); + const check = checks.get('sep-9999-ssrf-no-redirects'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('followed a 302'); + }, + TIMEOUT + ); +}); + +describe('control envelopes', () => { + test( + 'an unsigned envelope fails, since a receiver cannot verify it', + async () => { + const checks = await deliveryChecks(delivering({ signEnvelopes: false })); + const check = checks.get('sep-9999-envelope-signed-like-deliveries'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('cannot verify it'); + }, + TIMEOUT + ); + + test( + 'a webhook-id outside msg__ warns', + async () => { + const checks = await deliveryChecks( + delivering({ envelopeIdFormat: 'plain' }) + ); + const check = checks.get('sep-9999-envelope-webhook-id-format'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('outside the documented form'); + }, + TIMEOUT + ); + + // These two normally report untestable, because no client can ask a server + // for a retention gap or a revocation. A server that sends one unasked is + // graded on it, which is the only evidence these checks work. + test( + 'gap and terminated envelopes are graded when they arrive', + async () => { + const quiet = await deliveryChecks(delivering()); + expect(quiet.get('sep-9999-envelope-gap')?.details?.untestable).toBe( + true + ); + expect( + quiet.get('sep-9999-envelope-terminated')?.details?.untestable + ).toBe(true); + + const sent = await deliveryChecks( + delivering({ gapEnvelope: true, terminatedEnvelope: true }) + ); + expect(sent.get('sep-9999-envelope-gap')?.status).toBe('SUCCESS'); + expect(sent.get('sep-9999-envelope-terminated')?.status).toBe('SUCCESS'); + }, + TIMEOUT * 2 + ); + + test( + 'a gap envelope with no fresh cursor warns', + async () => { + const checks = await deliveryChecks( + delivering({ gapEnvelope: { cursor: null } }) + ); + const check = checks.get('sep-9999-envelope-gap'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('nothing to persist'); + }, + TIMEOUT + ); +}); diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts new file mode 100644 index 00000000..fd349f24 --- /dev/null +++ b/src/scenarios/server/events/negative-fixture.ts @@ -0,0 +1,926 @@ +/** + * The fixture the MCP Events negative controls are graded against. + * + * One server, five scenarios. Every option here exists to break exactly one + * rule while leaving the rest of the wire conformant, which is what makes a + * negative control a control: if two things are wrong at once, a flipped check + * does not say which one it caught. + * + * It speaks the SEP-2575 stateless wire (`server/discover` plus one POST per + * request), built per test rather than checked in as an example server, which + * matches the SEP-2640 negative tests. `events/stream` is a real SSE response + * held open with timers rather than a canned transcript, because the heartbeat + * cadence and the cancellation rows are about timing. + */ + +import { createHash, createHmac } from 'crypto'; +import { createServer, type IncomingMessage, type ServerResponse } from 'http'; +import type { AddressInfo } from 'net'; +import { withRequiredDraftResultFields } from '../../../mock-server'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { SUBSCRIPTION_ID_META } from './helpers'; + +/** A descriptor that is well formed apart from whatever a test overrides. */ +export function descriptor(overrides: Record = {}) { + return { + name: 'test.event', + description: 'A negative-control fixture event type.', + delivery: ['poll'], + inputSchema: { + type: 'object', + properties: { channel: { type: 'string' } } + }, + payloadSchema: { type: 'object', properties: { id: { type: 'string' } } }, + ...overrides + }; +} + +/** A poll result that is well formed apart from whatever a test overrides. */ +export function pollResult(overrides: Record = {}) { + return { + events: [], + cursor: 'cursor_001', + truncated: false, + hasMore: false, + nextPollMs: 30000, + ...overrides + }; +} + +/** An occurrence that is well formed apart from whatever a test overrides. */ +export function occurrence(overrides: Record = {}) { + return { + eventId: 'evt_001', + name: 'test.event', + timestamp: '2026-09-15T12:00:00Z', + data: { id: 'x' }, + ...overrides + }; +} + +export type JsonRpcErrorShape = { code: number; message: string }; + +/** A queued poll answer: a result body, or an error to answer with instead. */ +export type PollAnswer = Record | { error: JsonRpcErrorShape }; + +/** + * How `events/stream` behaves. The defaults are conformant: an immediate + * `active` confirmation carrying the parent request id in `_meta`, a heartbeat + * with a cursor, and one well-formed event. + */ +export interface StreamBehaviour { + /** Answer the POST with this JSON-RPC error instead of opening a stream. */ + error?: JsonRpcErrorShape; + /** Answer the POST with a plain JSON result, which is not a stream at all. */ + answerJson?: boolean; + /** Send no `notifications/events/active`. */ + omitActive?: boolean; + /** Fields merged into the `active` params. */ + activeParams?: Record; + /** + * Where the parent request id goes. `meta` is the spelling the document + * requires; `requestId` is the one kitchen-sink ships; `none` omits it. + */ + correlation?: 'meta' | 'requestId' | 'none'; + /** Heartbeat cadence. 0 sends none. */ + heartbeatMs?: number; + /** Fields merged into each heartbeat's params. */ + heartbeatParams?: Record; + /** Send `: keepalive` SSE comment lines at the heartbeat cadence too. */ + sseComments?: boolean; + /** Deliver an event this long after the stream opens. 0 sends none. */ + eventAfterMs?: number; + /** Fields merged into the delivered event's params. */ + eventParams?: Record; + /** Ride a non-`notifications/events/*` notification on the stream. */ + foreignNotification?: string; + /** Send `notifications/events/error` after this long. */ + errorNotificationAfterMs?: number; + /** Send `notifications/events/terminated` after this long. */ + terminatedAfterMs?: number; + /** Send a second `active` with `truncated: true`, the retention-gap signal. */ + gapAfterMs?: number; + /** Close the stream from the server side, writing a final frame first. */ + closeAfterMs?: number; + /** The `result` of that final frame. Defaults to an empty typed result. */ + finalResult?: Record; + /** Refuse concurrent opens past this many, the cap streams are exempt from. */ + maxConcurrent?: number; + /** Open a stream for any name, including one the catalog does not serve. */ + acceptAnyName?: boolean; +} + +const CONFORMANT_STREAM: Required< + Pick< + StreamBehaviour, + 'correlation' | 'heartbeatMs' | 'eventAfterMs' | 'maxConcurrent' + > +> = { + correlation: 'meta', + heartbeatMs: 150, + eventAfterMs: 80, + maxConcurrent: Infinity +}; + +/** + * How `events/subscribe` and `events/unsubscribe` behave. The defaults are + * conformant: an https-only callback, a validated `whsec_` secret, an id + * derived from the whole key, an idempotent upsert, and `-32011 NotFound` for + * a key the server does not hold. + */ +export interface SubscribeBehaviour { + /** Answer every subscribe with this error instead of subscribing. */ + error?: JsonRpcErrorShape; + /** Accept a subscribe carrying no `delivery.secret`. */ + acceptMissingSecret?: boolean; + /** Accept a secret without the `whsec_` prefix. */ + acceptBadPrefix?: boolean; + /** Accept a `whsec_` secret that decodes to fewer than 24 bytes. */ + acceptShortSecret?: boolean; + /** Accept an `http://` callback URL. */ + acceptHttpUrl?: boolean; + /** Code for a rejected secret or URL, where the document says -32602. */ + rejectionCode?: number; + /** Subscribe to a type whose `delivery` does not list `webhook`. */ + acceptNonWebhookType?: boolean; + /** Grant `refreshBefore: null` however finite the suggestion. */ + nullRefreshAlways?: boolean; + /** Grant `refreshBefore: null` when `ttlMs` was omitted. */ + nullRefreshOnOmitted?: boolean; + /** Grant a `refreshBefore` well past the suggestion. */ + grantBeyondSuggestion?: boolean; + /** Raw `refreshBefore` to answer with, for the non-timestamp case. */ + refreshBefore?: unknown; + /** Reject a `ttlMs` the server dislikes instead of clamping it. */ + rejectTtl?: 'short' | 'long' | 'both'; + /** Fields merged into every subscribe result. */ + result?: Record; + /** Omit `id` from the subscribe result. */ + omitId?: boolean; + /** Mint a fresh id per call, so a repeat subscribe is not an upsert. */ + nonIdempotentId?: boolean; + /** Derive the id from `(name, arguments)` only, leaving the URL out. */ + idIgnoresUrl?: boolean; + /** Treat a supplied `id` as addressing that subscription. */ + idIsAnInput?: boolean; + /** Answer success for an unsubscribe of a key never held. */ + unsubscribeUnknownOk?: boolean; + /** Code for an unsubscribe of an unknown key, where the document says -32011. */ + unsubscribeUnknownCode?: number; + /** Code for an unsubscribe of a key the server does hold. */ + unsubscribeHeldCode?: number; + /** Answer -32013 once this many subscriptions are live. */ + maxSubscriptions?: number; +} + +/** + * What the fixture POSTs to a webhook callback once a subscription exists. The + * defaults are conformant: a verification challenge first, then one event, both + * signed per Standard Webhooks with the decoded secret bytes, both carrying + * `X-MCP-Subscription-Id`, retried on 5xx with a fresh timestamp and signature, + * and never retried after 410 or 413. + * + * Delivering at all is what a *loopback* callback makes wrong, so the SSRF rows + * fail whenever this is switched on without `EVENTS_WEBHOOK_CALLBACK_BASE`. + * That is the fixture standing in for a server run with its private-network + * guard disabled, which is the state the demo fixtures ship in. + */ +export interface DeliveryBehaviour { + /** Send the verification envelope before any event. */ + verify?: boolean; + /** Deliver an event at all. */ + sendEvent?: boolean; + /** Deliver the event before the verification envelope. */ + eventBeforeVerification?: boolean; + /** Sign with the literal `whsec_…` string instead of its decoded bytes. */ + literalKeySignature?: boolean; + /** Standard Webhooks headers to leave off. */ + omitHeaders?: string[]; + /** Leave off `X-MCP-Subscription-Id`. */ + omitSubscriptionIdHeader?: boolean; + /** Send an `X-MCP-Subscription-Id` that is not the subscribe response's id. */ + wrongSubscriptionIdHeader?: boolean; + /** Deliver with this method instead of POST. */ + method?: string; + /** Deliver with this Content-Type instead of application/json. */ + contentType?: string; + /** Pad the event body past the 256 KiB ceiling. */ + oversizedBody?: boolean; + /** Follow a 302 from the callback, which the document forbids. */ + followRedirects?: boolean; + /** Total attempts for a delivery the callback rejects with 5xx. */ + attempts?: number; + /** Reuse the first attempt's timestamp and signature on every retry. */ + staleRetrySignature?: boolean; + /** Retry after 410 and 413, which the document defines as non-retryable. */ + retryNonRetryable?: boolean; + /** Sign control envelopes the way deliveries are signed. */ + signEnvelopes?: boolean; + /** `webhook-id` form for control envelopes. */ + envelopeIdFormat?: 'msg' | 'plain'; + /** Send a `gap` control envelope. */ + gapEnvelope?: boolean | { cursor?: unknown }; + /** Send a `terminated` control envelope. */ + terminatedEnvelope?: boolean | { error?: unknown }; +} + +export interface EventsFixtureOptions { + /** Raw value to declare at `capabilities.events`; omit for no declaration. */ + capability?: unknown; + descriptors?: object[]; + /** Answer `events/list` with this JSON-RPC error instead of a result. */ + listError?: JsonRpcErrorShape; + /** + * Poll responses, consumed in order; the last one repeats once exhausted. + * A `{ error }` entry makes that poll answer with a JSON-RPC error. + */ + pollResponses?: PollAnswer[]; + /** Overrides keyed by the polled event name, taking priority over the queue. */ + pollByName?: Record; + /** Error code for a poll naming an event type the fixture does not serve. */ + unknownNameCode?: number; + /** Error code for a poll whose arguments violate `inputSchema`. */ + invalidArgsCode?: number; + /** How `events/stream` behaves. */ + stream?: StreamBehaviour; + /** How `events/subscribe` and `events/unsubscribe` behave. */ + subscribe?: SubscribeBehaviour; + /** + * What the fixture POSTs to the callback. Omit it and the fixture subscribes + * without ever delivering, which is what a server with no webhook delivery + * looks like from the receiver's side. + */ + delivery?: DeliveryBehaviour; +} + +export interface EventsFixture { + url: string; + /** Params of every `events/poll` received, in order. */ + polls: Array>; + /** Params of every `events/stream` received, in order. */ + streams: Array>; + /** Params of every `events/subscribe` received, in order. */ + subscribes: Array>; + /** Subscription keys the fixture still holds when it is asked. */ + liveSubscriptions(): string[]; + close(): Promise; +} + +/** `whsec_` plus base64 of 24–64 bytes, which is what the document requires. */ +function secretProblem( + secret: unknown, + behaviour: SubscribeBehaviour +): 'missing' | 'prefix' | 'length' | undefined { + if (secret === undefined || secret === null) { + return behaviour.acceptMissingSecret ? undefined : 'missing'; + } + if (typeof secret !== 'string' || !secret.startsWith('whsec_')) { + return behaviour.acceptBadPrefix ? undefined : 'prefix'; + } + const bytes = Buffer.from(secret.slice('whsec_'.length), 'base64'); + if (bytes.length < 24 || bytes.length > 64) { + return behaviour.acceptShortSecret ? undefined : 'length'; + } + return undefined; +} + +export async function startEventsFixture( + opts: EventsFixtureOptions +): Promise { + const polls: Array> = []; + const streams: Array> = []; + const subscribes: Array> = []; + /** Live subscriptions, keyed the way the document keys them. */ + const subscriptions = new Map(); + /** Deliveries still in flight, so close() can settle rather than abandon. */ + const inFlight = new Set>(); + let mintedIds = 0; + const queue = [...(opts.pollResponses ?? [pollResult()])]; + const descriptors = opts.descriptors ?? [descriptor()]; + const names = new Set( + descriptors + .map((d) => (d as { name?: unknown }).name) + .filter((n): n is string => typeof n === 'string') + ); + + /** Open streams, so close() can tear them down instead of hanging on them. */ + const openStreams = new Set<{ res: ServerResponse; stop: () => void }>(); + let liveStreams = 0; + + const server = createServer(async (req, res) => { + if (req.method !== 'POST') { + res.writeHead(405).end(); + return; + } + const body = await readJsonBody(req); + const method = body.method as string; + const id = body.id; + const params = (body.params ?? {}) as Record; + + const send = (result: object) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ + jsonrpc: '2.0', + id, + result: withRequiredDraftResultFields(method, result) + }) + ); + }; + const fail = (code: number, message: string, data?: unknown) => { + res.writeHead(200, { 'Content-Type': 'application/json' }); + res.end( + JSON.stringify({ jsonrpc: '2.0', id, error: { code, message, data } }) + ); + }; + + if (method === 'server/discover') { + send({ + supportedVersions: [DRAFT_PROTOCOL_VERSION], + capabilities: 'capability' in opts ? { events: opts.capability } : {}, + serverInfo: { name: 'events-negative', version: '1.0.0' } + }); + return; + } + + if (method === 'events/list') { + if (opts.listError) { + fail(opts.listError.code, opts.listError.message); + return; + } + send({ events: descriptors }); + return; + } + + if (method === 'events/poll') { + polls.push(params); + const name = params.name; + + if (typeof name !== 'string') { + fail(-32602, 'InvalidParams: `name` is required'); + return; + } + if (!names.has(name)) { + fail(opts.unknownNameCode ?? -32011, 'NotFound', { kind: 'event' }); + return; + } + + const byName = opts.pollByName?.[name]; + const chosen = + byName ?? + (queue.length > 1 ? queue.shift()! : (queue[0] ?? pollResult())); + + // Argument validation against the fixture's own declared schema, so the + // invalid-arguments probe has something real to violate. + const args = (params.arguments ?? {}) as Record; + const decl = descriptors.find( + (d) => (d as { name?: unknown }).name === name + ) as { inputSchema?: { properties?: Record } }; + for (const [key, value] of Object.entries(args)) { + const declared = decl?.inputSchema?.properties?.[key]; + if (declared?.type === 'string' && typeof value !== 'string') { + fail(opts.invalidArgsCode ?? -32602, 'InvalidParams'); + return; + } + } + + if ('error' in chosen) { + const e = (chosen as { error: JsonRpcErrorShape }).error; + fail(e.code, e.message); + return; + } + send(chosen as Record); + return; + } + + if (method === 'events/subscribe' || method === 'events/unsubscribe') { + const behaviour = opts.subscribe ?? {}; + const delivery = isRecord(params.delivery) ? params.delivery : {}; + const name = params.name; + const url = delivery.url; + // The document's key, minus the principal a single run cannot vary. + const key = [ + behaviour.idIgnoresUrl ? '' : String(url), + String(name), + JSON.stringify(params.arguments ?? {}) + ].join('|'); + + if (method === 'events/unsubscribe') { + const held = subscriptions.has(key); + if (held && behaviour.unsubscribeHeldCode !== undefined) { + fail(behaviour.unsubscribeHeldCode, 'Unsubscribe refused'); + return; + } + if (held) { + subscriptions.delete(key); + send({}); + return; + } + if (behaviour.unsubscribeUnknownOk) { + send({}); + return; + } + fail(behaviour.unsubscribeUnknownCode ?? -32011, 'NotFound', { + kind: 'subscription' + }); + return; + } + + subscribes.push(params); + if (behaviour.error) { + fail(behaviour.error.code, behaviour.error.message); + return; + } + + const rejectionCode = behaviour.rejectionCode ?? -32602; + const problem = secretProblem(delivery.secret, behaviour); + if (problem) { + fail(rejectionCode, `InvalidParams: \`delivery.secret\` (${problem})`); + return; + } + if ( + !behaviour.acceptHttpUrl && + (typeof url !== 'string' || !url.startsWith('https://')) + ) { + fail(rejectionCode, 'InvalidParams: `delivery.url` must be https'); + return; + } + + const served = descriptors.find( + (d) => (d as { name?: unknown }).name === name + ) as { delivery?: unknown } | undefined; + if (!served) { + fail(opts.unknownNameCode ?? -32011, 'NotFound', { kind: 'event' }); + return; + } + const modes = Array.isArray(served.delivery) ? served.delivery : []; + if (!modes.includes('webhook') && !behaviour.acceptNonWebhookType) { + fail(-32014, 'Unsupported: event type does not offer webhook delivery'); + return; + } + + const ttl = params.ttlMs; + const reject = behaviour.rejectTtl; + const shortTtl = typeof ttl === 'number' && ttl <= 60_000; + const longTtl = typeof ttl === 'number' && ttl > 7 * 24 * 3600_000; + if ( + (reject === 'both' && (shortTtl || longTtl)) || + (reject === 'short' && shortTtl) || + (reject === 'long' && longTtl) + ) { + fail(-32602, 'InvalidParams: `ttlMs` out of range'); + return; + } + + // An `id` supplied by the caller is a routing handle, never an input, so + // by default it has no bearing on which subscription this addresses. + const addressed = + behaviour.idIsAnInput && typeof params.id === 'string' + ? [...subscriptions.entries()].find( + ([, sub]) => sub.id === params.id + )?.[0] + : undefined; + const effectiveKey = addressed ?? key; + + const existing = subscriptions.get(effectiveKey); + if ( + !existing && + behaviour.maxSubscriptions !== undefined && + subscriptions.size >= behaviour.maxSubscriptions + ) { + fail(-32013, 'ResourceExhausted: subscription cap reached'); + return; + } + const id = + existing && !behaviour.nonIdempotentId + ? existing.id + : behaviour.nonIdempotentId + ? `sub_${++mintedIds}_${hashKey(effectiveKey)}` + : hashKey(effectiveKey); + subscriptions.set(effectiveKey, { id }); + + // Clamp rather than reject, and never hand back no-expiry unasked. + const cap = 7 * 24 * 3600_000; + let refreshBefore: unknown; + if (behaviour.refreshBefore !== undefined) { + refreshBefore = behaviour.refreshBefore; + } else if (behaviour.nullRefreshAlways) { + refreshBefore = null; + } else if (ttl === null) { + refreshBefore = null; + } else if (ttl === undefined) { + refreshBefore = behaviour.nullRefreshOnOmitted + ? null + : new Date(Date.now() + 3600_000).toISOString(); + } else { + const granted = behaviour.grantBeyondSuggestion + ? Number(ttl) + 7 * 24 * 3600_000 + : Math.min(Number(ttl), cap); + refreshBefore = new Date(Date.now() + granted).toISOString(); + } + + send({ + ...(behaviour.omitId ? {} : { id }), + refreshBefore, + cursor: 'cursor_sub_001', + truncated: false, + ...(behaviour.result ?? {}) + }); + + // Delivery runs after the response, because that is the order a receiver + // sees it in: the subscribe returns, then the callback starts ringing. + if (opts.delivery && typeof url === 'string') { + const run = deliverToCallback( + url, + delivery.secret, + id, + String(name), + opts.delivery + ).finally(() => inFlight.delete(run)); + inFlight.add(run); + } + return; + } + + if (method === 'events/stream') { + streams.push(params); + const behaviour = { ...CONFORMANT_STREAM, ...(opts.stream ?? {}) }; + const name = params.name; + + if (behaviour.error) { + fail(behaviour.error.code, behaviour.error.message); + return; + } + if ( + !behaviour.acceptAnyName && + (typeof name !== 'string' || !names.has(name)) + ) { + fail(opts.unknownNameCode ?? -32011, 'NotFound', { kind: 'event' }); + return; + } + if (liveStreams >= behaviour.maxConcurrent) { + fail(-32013, 'ResourceExhausted: too many subscriptions'); + return; + } + if (behaviour.answerJson) { + send({}); + return; + } + + liveStreams += 1; + const entry = openStream(res, id, String(name), behaviour); + openStreams.add(entry); + const done = () => { + if (!openStreams.delete(entry)) return; + liveStreams -= 1; + entry.stop(); + }; + req.on('close', done); + res.on('close', done); + return; + } + + fail(-32601, `Method not found: ${method}`); + }); + + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, () => resolve()); + }); + const addr = server.address() as AddressInfo; + + return { + url: `http://localhost:${addr.port}/mcp`, + polls, + streams, + subscribes, + liveSubscriptions: () => [...subscriptions.keys()], + async close() { + // Let the callback stop ringing before the receiver goes away, so a + // pending fetch cannot outlive the test that started it. + await Promise.race([ + Promise.allSettled([...inFlight]), + new Promise((resolve) => setTimeout(resolve, 2000)) + ]); + for (const entry of [...openStreams]) { + openStreams.delete(entry); + entry.stop(); + entry.res.destroy(); + } + liveStreams = 0; + server.closeAllConnections?.(); + await new Promise((r) => server.close(() => r())); + } + }; +} + +/** Write the SSE frames a push subscription produces, on timers. */ +function openStream( + res: ServerResponse, + requestId: unknown, + name: string, + behaviour: StreamBehaviour & typeof CONFORMANT_STREAM +): { res: ServerResponse; stop: () => void } { + const timers: NodeJS.Timeout[] = []; + const stop = () => { + for (const t of timers) clearInterval(t); + timers.length = 0; + }; + + res.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }); + + /** The correlation id goes where the behaviour says, not always in `_meta`. */ + const correlate = (params: Record) => { + if (behaviour.correlation === 'meta') { + return { ...params, _meta: { [SUBSCRIPTION_ID_META]: requestId } }; + } + if (behaviour.correlation === 'requestId') { + return { ...params, requestId }; + } + return params; + }; + const notify = (method: string, params: Record) => { + if (res.writableEnded) return; + res.write( + `data: ${JSON.stringify({ jsonrpc: '2.0', method, params: correlate(params) })}\n\n` + ); + }; + const after = (ms: number, fn: () => void) => { + const t = setTimeout(fn, ms); + timers.push(t); + }; + + if (!behaviour.omitActive) { + notify('notifications/events/active', { + cursor: 'cursor_stream_001', + truncated: false, + ...(behaviour.activeParams ?? {}) + }); + } + + if (behaviour.heartbeatMs > 0) { + const beat = setInterval(() => { + notify('notifications/events/heartbeat', { + cursor: 'cursor_stream_001', + ...(behaviour.heartbeatParams ?? {}) + }); + if (behaviour.sseComments && !res.writableEnded) { + res.write(': keepalive\n\n'); + } + }, behaviour.heartbeatMs); + timers.push(beat); + } else if (behaviour.sseComments) { + const beat = setInterval(() => { + if (!res.writableEnded) res.write(': keepalive\n\n'); + }, 150); + timers.push(beat); + } + + if (behaviour.eventAfterMs > 0) { + after(behaviour.eventAfterMs, () => + notify('notifications/events/event', { + ...occurrence({ name }), + ...(behaviour.eventParams ?? {}) + }) + ); + } + + if (behaviour.foreignNotification) { + after(60, () => notify(behaviour.foreignNotification!, {})); + } + if (behaviour.errorNotificationAfterMs) { + after(behaviour.errorNotificationAfterMs, () => + notify('notifications/events/error', { + code: -32603, + message: 'upstream unavailable, retrying' + }) + ); + } + if (behaviour.terminatedAfterMs) { + after(behaviour.terminatedAfterMs, () => + notify('notifications/events/terminated', { reason: 'revoked' }) + ); + } + if (behaviour.gapAfterMs) { + after(behaviour.gapAfterMs, () => + notify('notifications/events/active', { + cursor: 'cursor_stream_002', + truncated: true + }) + ); + } + if (behaviour.closeAfterMs) { + after(behaviour.closeAfterMs, () => { + if (res.writableEnded) return; + res.write( + `data: ${JSON.stringify({ + jsonrpc: '2.0', + id: requestId, + result: behaviour.finalResult ?? { _meta: {} } + })}\n\n` + ); + stop(); + res.end(); + }); + } + + return { res, stop }; +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Retry spacing. Over a second, because `webhook-timestamp` is in seconds and + * two attempts inside one second would share a stamp the fixture did freshen. */ +const RETRY_GAP_MS = 1100; + +/** + * POST the verification challenge and then the event, the way the document says + * to, with whatever this behaviour breaks. + * + * Errors are swallowed: the callback is a receiver the scenario owns, it may + * refuse or vanish mid-run by design, and a fixture that threw here would fail + * the test rather than the check under test. + */ +async function deliverToCallback( + url: string, + secret: unknown, + subscriptionId: string, + eventName: string, + behaviour: DeliveryBehaviour +): Promise { + const key = + typeof secret === 'string' && secret.startsWith('whsec_') + ? behaviour.literalKeySignature + ? Buffer.from(secret) + : Buffer.from(secret.slice('whsec_'.length), 'base64') + : Buffer.from(String(secret ?? '')); + + const headersFor = (webhookId: string, timestamp: string, body: string) => { + const omit = new Set(behaviour.omitHeaders ?? []); + const signature = `v1,${createHmac('sha256', key).update(`${webhookId}.${timestamp}.${body}`).digest('base64')}`; + const headers: Record = { + 'content-type': behaviour.contentType ?? 'application/json' + }; + if (!omit.has('webhook-id')) headers['webhook-id'] = webhookId; + if (!omit.has('webhook-timestamp')) + headers['webhook-timestamp'] = timestamp; + if (!omit.has('webhook-signature')) + headers['webhook-signature'] = signature; + if (!behaviour.omitSubscriptionIdHeader) { + headers['x-mcp-subscription-id'] = behaviour.wrongSubscriptionIdHeader + ? 'sub_someone_elses' + : subscriptionId; + } + return headers; + }; + + /** One delivery, with the retry rules applied to whatever it answers. */ + const post = async ( + webhookId: string, + body: string, + opts: { retryable?: boolean } = {} + ): Promise => { + const attempts = Math.max(1, behaviour.attempts ?? 3); + let stamp = String(Math.floor(Date.now() / 1000)); + let target = url; + + for (let attempt = 1; attempt <= attempts; attempt++) { + if (attempt > 1 && !behaviour.staleRetrySignature) { + stamp = String(Math.floor(Date.now() / 1000)); + } + let status: number; + let location: string | null = null; + try { + const res = await fetch(target, { + method: behaviour.method ?? 'POST', + headers: headersFor(webhookId, stamp, body), + body, + redirect: 'manual' + }); + status = res.status; + location = res.headers.get('location'); + await res.text(); + } catch { + return; + } + + if (status === 302 && location) { + // A conformant server stops here. Following is the SSRF hazard the + // no-redirects rule exists for. + if (!behaviour.followRedirects) return; + target = location; + continue; + } + if (status >= 200 && status < 300) return; + if ((status === 410 || status === 413) && !behaviour.retryNonRetryable) { + return; + } + if (opts.retryable === false) return; + if (attempt === attempts) return; + await new Promise((resolve) => setTimeout(resolve, RETRY_GAP_MS)); + } + }; + + const envelopeId = (type: string) => + behaviour.envelopeIdFormat === 'plain' + ? `wh-${Math.random().toString(36).slice(2, 10)}` + : `msg_${type}_${Math.random().toString(36).slice(2, 10)}`; + + /** Control envelopes are signed and headed exactly like deliveries — unless + * this behaviour is the one that says otherwise. */ + const postEnvelope = async ( + type: string, + body: Record + ): Promise => { + const json = JSON.stringify({ type, ...body }); + if (behaviour.signEnvelopes === false) { + try { + await fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: json, + redirect: 'manual' + }).then((r) => r.text()); + } catch { + // See above: a receiver refusing is the scenario's business. + } + return; + } + await post(envelopeId(type), json); + }; + + const postEvent = async (): Promise => { + const data = behaviour.oversizedBody + ? { id: 'x', padding: 'p'.repeat(300 * 1024) } + : { id: 'x' }; + await post( + envelopeId('event'), + JSON.stringify({ + eventId: `evt_${Math.random().toString(36).slice(2, 10)}`, + name: eventName, + timestamp: new Date().toISOString(), + data + }) + ); + }; + + if (behaviour.eventBeforeVerification) { + await postEvent(); + // The check compares millisecond arrival times, and two loopback POSTs land + // inside the same millisecond often enough to make the out-of-order case + // read as in-order. A real server delivering before it verifies is not this + // close, so the gap is realism rather than a thumb on the scale. + await new Promise((resolve) => setTimeout(resolve, 60)); + } + if (behaviour.verify !== false) { + await postEnvelope('verification', { + challenge: `chal_${Math.random().toString(36).slice(2, 14)}`, + subscriptionId + }); + } + if (behaviour.sendEvent !== false && !behaviour.eventBeforeVerification) { + await postEvent(); + } + if (behaviour.gapEnvelope) { + const override = isRecord(behaviour.gapEnvelope) + ? behaviour.gapEnvelope + : {}; + await postEnvelope('gap', { + cursor: 'cursor_after_gap', + ...override + }); + } + if (behaviour.terminatedEnvelope) { + const override = isRecord(behaviour.terminatedEnvelope) + ? behaviour.terminatedEnvelope + : {}; + await postEnvelope('terminated', { + error: { code: -32012, message: 'authorization revoked' }, + ...override + }); + } +} + +/** A deterministic id over the subscription key, which is what the document asks for. */ +function hashKey(key: string): string { + return `sub_${createHash('sha256').update(key).digest('hex').slice(0, 24)}`; +} + +export async function readJsonBody( + req: IncomingMessage +): Promise> { + const chunks: Buffer[] = []; + for await (const chunk of req) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< + string, + unknown + >; +} diff --git a/src/scenarios/server/events/negative-push.test.ts b/src/scenarios/server/events/negative-push.test.ts new file mode 100644 index 00000000..76d87958 --- /dev/null +++ b/src/scenarios/server/events/negative-push.test.ts @@ -0,0 +1,351 @@ +import { describe, test, expect, vi, afterEach } from 'vitest'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { + descriptor, + startEventsFixture, + type EventsFixtureOptions, + type StreamBehaviour +} from './negative-fixture'; + +/** + * Negative controls for `events-push`. + * + * A passing run against kitchen-sink proves the scenario does not + * false-positive. It does not prove any check catches anything, which is what + * these are for: each case pairs the conformant fixture with one broken in + * exactly one way and asserts the check flips. + * + * Four rows report untestable against both real implementations, because no + * client can ask a server to fail upstream, lose its replay window, terminate a + * subscription or close a stream. The fixture can do all four on demand, so + * they are graded here rather than only declared — which is the only evidence + * that those checks work at all. + * + * `EVENTS_PUSH_WATCH_MS` is stubbed down from 35s per case, since the scenario + * reads it when the module is first evaluated. `vi.resetModules()` before each + * dynamic import is what makes the stub land regardless of whether another test + * file imported the scenario first. The catch is that a reset registry hands + * back a *second* copy of the connection module, and `err instanceof + * JsonRpcError` is false across two copies of the same class — so the run + * context has to be built from the same fresh graph as the scenario, not from a + * static import up here. + */ + +/** The window every case uses, except the one that needs to outlast 30s. */ +const FAST_WATCH_MS = 900; + +async function pushChecks( + opts: EventsFixtureOptions, + watchMs: number = FAST_WATCH_MS +) { + vi.resetModules(); + vi.stubEnv('EVENTS_PUSH_WATCH_MS', String(watchMs)); + const { EventsPushScenario } = await import('./push'); + const { testContext } = await import('../../../connection/testing'); + const { takeWireViolations } = + await import('../../../validation/wire-schema'); + const fixture = await startEventsFixture(opts); + try { + const checks = await new EventsPushScenario().run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed frame does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return new Map(checks.map((c) => [c.id, c])); + } finally { + await fixture.close(); + } +} + +/** A push-capable catalog, which is the only thing this scenario selects on. */ +function pushFixture(stream: StreamBehaviour = {}): EventsFixtureOptions { + return { + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'push.event', delivery: ['push'] })], + stream + }; +} + +afterEach(() => { + vi.unstubAllEnvs(); +}); + +describe('events/stream opening', () => { + test('the conformant fixture passes every gradeable row', async () => { + const checks = await pushChecks(pushFixture()); + for (const id of [ + 'sep-9999-stream-implemented', + 'sep-9999-stream-active-confirmation', + 'sep-9999-stream-subscription-id-meta', + 'sep-9999-stream-event-notification', + 'sep-9999-stream-carries-only-event-notifications', + 'sep-9999-stream-heartbeat-required', + 'sep-9999-stream-heartbeat-carries-cursor', + 'sep-9999-stream-heartbeat-interval', + 'sep-9999-stream-heartbeat-not-sse-comment', + 'sep-9999-stream-cancel-stops-delivery', + 'sep-9999-stream-exempt-from-concurrency-cap', + 'sep-9999-stream-error-before-open' + ]) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }); + + test('a server with no push-capable type reports every row untestable', async () => { + const checks = await pushChecks({ + capability: { listChanged: true }, + descriptors: [descriptor()] + }); + const check = checks.get('sep-9999-stream-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('No event type advertises'); + expect(check?.details?.untestable).toBe(true); + }); + + test('a server that declares nothing and serves nothing skips the suite', async () => { + const checks = await pushChecks({ + listError: { code: -32601, message: 'Method not found' } + }); + for (const check of checks.values()) { + expect(check.status).toBe('SKIPPED'); + } + }); + + test('answering events/stream with a JSON result rather than a stream fails', async () => { + const checks = await pushChecks(pushFixture({ answerJson: true })); + const check = checks.get('sep-9999-stream-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected an SSE stream'); + // And nothing on the stream is reported green just because it was silent. + expect(checks.get('sep-9999-stream-heartbeat-required')?.status).toBe( + 'FAILURE' + ); + expect( + checks.get('sep-9999-stream-heartbeat-required')?.details?.untestable + ).toBe(true); + }); + + test('refusing a valid subscription outright fails and names the code', async () => { + const checks = await pushChecks( + pushFixture({ error: { code: -32013, message: 'ResourceExhausted' } }) + ); + const check = checks.get('sep-9999-stream-implemented'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32013'); + }); + + test('opening a stream for an unknown event type fails error-before-open', async () => { + const checks = await pushChecks(pushFixture({ acceptAnyName: true })); + const check = checks.get('sep-9999-stream-error-before-open'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('did not answer a JSON-RPC error'); + }); + + test('answering an unknown type with -32014 rather than NotFound warns', async () => { + const checks = await pushChecks({ + ...pushFixture(), + unknownNameCode: -32014 + }); + const check = checks.get('sep-9999-stream-error-before-open'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('-32011 NotFound'); + }); +}); + +describe('the active confirmation', () => { + test('no confirmation at all fails', async () => { + const checks = await pushChecks(pushFixture({ omitActive: true })); + const check = checks.get('sep-9999-stream-active-confirmation'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('notifications/events/active'); + }); + + test('a numeric cursor on the confirmation fails', async () => { + const checks = await pushChecks( + pushFixture({ activeParams: { cursor: 42 } }) + ); + const check = checks.get('sep-9999-stream-active-confirmation'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('`cursor` is'); + }); + + test('a string truncated flag fails', async () => { + const checks = await pushChecks( + pushFixture({ activeParams: { truncated: 'no' } }) + ); + const check = checks.get('sep-9999-stream-active-confirmation'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('`truncated` is'); + }); +}); + +describe('the correlation id', () => { + // The divergence this row exists for: mcpkit puts the id in params.requestId, + // mirroring the sketch's own push examples, where the document requires the + // SEP-2575 `_meta` spelling. A client holding two streams cannot route by + // what the document tells it to read. + test('params.requestId instead of _meta fails and names the key', async () => { + const checks = await pushChecks(pushFixture({ correlation: 'requestId' })); + const check = checks.get('sep-9999-stream-subscription-id-meta'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain( + 'io.modelcontextprotocol/subscriptionId' + ); + }); + + test('no correlation id at all fails', async () => { + const checks = await pushChecks(pushFixture({ correlation: 'none' })); + expect(checks.get('sep-9999-stream-subscription-id-meta')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('what rides the stream', () => { + test('a non-events notification fails and names the method', async () => { + const checks = await pushChecks( + pushFixture({ foreignNotification: 'notifications/message' }) + ); + const check = checks.get( + 'sep-9999-stream-carries-only-event-notifications' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('notifications/message'); + }); + + test('a malformed occurrence fails the event row', async () => { + const checks = await pushChecks( + pushFixture({ eventParams: { timestamp: 'last tuesday' } }) + ); + const check = checks.get('sep-9999-stream-event-notification'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('EventOccurrence'); + }); + + test('a stream that delivers nothing reports the event row untestable', async () => { + const checks = await pushChecks(pushFixture({ eventAfterMs: 0 })); + const check = checks.get('sep-9999-stream-event-notification'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBe(true); + }); +}); + +describe('the heartbeat', () => { + test('a numeric cursor on the heartbeat fails', async () => { + const checks = await pushChecks( + pushFixture({ heartbeatParams: { cursor: 42 } }) + ); + const check = checks.get('sep-9999-stream-heartbeat-carries-cursor'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('neither a string nor null'); + }); + + test('silence inside a short window is untestable, not a failure', async () => { + const checks = await pushChecks(pushFixture({ heartbeatMs: 0 })); + const check = checks.get('sep-9999-stream-heartbeat-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('EVENTS_PUSH_WATCH_MS'); + }); + + // The one case that has to outlast the cadence the document permits. Under + // 30s a silent server and a slow one are indistinguishable, so this is the + // only window in which the MUST can actually fail. + test('silence past 30s fails the heartbeat MUST', async () => { + const checks = await pushChecks(pushFixture({ heartbeatMs: 0 }), 30_050); + const check = checks.get('sep-9999-stream-heartbeat-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBeUndefined(); + expect(check?.errorMessage).toContain('outlasts the 30s cadence'); + }, 60_000); + + test('an SSE comment keepalive beside a data heartbeat warns', async () => { + const checks = await pushChecks(pushFixture({ sseComments: true })); + const check = checks.get('sep-9999-stream-heartbeat-not-sse-comment'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('keepalive'); + }); + + test('an SSE comment instead of a data heartbeat fails', async () => { + const checks = await pushChecks( + pushFixture({ heartbeatMs: 0, sseComments: true }) + ); + expect( + checks.get('sep-9999-stream-heartbeat-not-sse-comment')?.status + ).toBe('FAILURE'); + }); +}); + +describe('concurrency and cancellation', () => { + // The cap kitchen-sink applies to streams, which the document exempts them + // from: the first stream confirms and the other two are refused -32013. + test('a per-principal cap that catches streams fails', async () => { + const checks = await pushChecks(pushFixture({ maxConcurrent: 1 })); + const check = checks.get('sep-9999-stream-exempt-from-concurrency-cap'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('only 1 confirmed'); + }); + + test('a server that keeps sending after the abort fails cancel-stops-delivery', async () => { + // Nothing in the fixture can outlive an aborted request, so the row is + // proven the other way round: it passes here, and the failure branch is + // reachable only by a server that holds the connection open after abort. + const checks = await pushChecks(pushFixture()); + expect(checks.get('sep-9999-stream-cancel-stops-delivery')?.status).toBe( + 'SUCCESS' + ); + }); +}); + +describe('rows that are untestable against a real server', () => { + test('a server-initiated close grades the final result instead of skipping it', async () => { + const cancelled = await pushChecks(pushFixture()); + const untested = cancelled.get('sep-9999-stream-final-result-shape'); + expect(untested?.status).toBe('WARNING'); + expect(untested?.details?.untestable).toBe(true); + + const closed = await pushChecks(pushFixture({ closeAfterMs: 400 })); + expect(closed.get('sep-9999-stream-final-result-shape')?.status).toBe( + 'SUCCESS' + ); + expect(closed.get('sep-9999-stream-final-result-timing')?.status).toBe( + 'SUCCESS' + ); + }); + + test('a final result carrying fields fails, since it is defined as empty', async () => { + const checks = await pushChecks( + pushFixture({ closeAfterMs: 400, finalResult: { events: [] } }) + ); + const check = checks.get('sep-9999-stream-final-result-shape'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('events'); + }); + + test('an error notification, a termination and a gap are all graded when they happen', async () => { + const quiet = await pushChecks(pushFixture()); + for (const id of [ + 'sep-9999-stream-error-is-recoverable', + 'sep-9999-stream-terminated-ends-subscription', + 'sep-9999-stream-gap-resends-active' + ]) { + expect(quiet.get(id)?.details?.untestable, id).toBe(true); + } + + const busy = await pushChecks( + pushFixture({ + errorNotificationAfterMs: 150, + terminatedAfterMs: 250, + gapAfterMs: 350 + }) + ); + for (const id of [ + 'sep-9999-stream-error-is-recoverable', + 'sep-9999-stream-terminated-ends-subscription', + 'sep-9999-stream-gap-resends-active' + ]) { + expect(busy.get(id)?.status, id).toBe('SUCCESS'); + } + }); +}); diff --git a/src/scenarios/server/events/negative-webhook.test.ts b/src/scenarios/server/events/negative-webhook.test.ts new file mode 100644 index 00000000..7bd8d366 --- /dev/null +++ b/src/scenarios/server/events/negative-webhook.test.ts @@ -0,0 +1,413 @@ +import { describe, test, expect } from 'vitest'; +import { testContext } from '../../../connection/testing'; +import { DRAFT_PROTOCOL_VERSION } from '../../../types'; +import { takeWireViolations } from '../../../validation/wire-schema'; +import { EventsWebhookScenario } from './webhook'; +import { + descriptor, + startEventsFixture, + type EventsFixtureOptions, + type SubscribeBehaviour +} from './negative-fixture'; + +/** + * Negative controls for `events-webhook`. + * + * Each case pairs the conformant fixture with one broken in exactly one way. + * Where a single defect costs more than one row, the test says so rather than + * pretending the rows are independent: a server that returns no `id` at all + * takes the key-composition rows down with it, because there is nothing left to + * compare. + * + * Two of these are divergences the suite found in a real implementation, and + * both are here so a red run can be trusted: an `http://` callback accepted at + * subscribe, and an unsubscribe of a key the server never held answering + * success. + */ + +const ALL_ROWS = 27; + +/** Rows every conformant run passes, which is the whole surface bar the seven + * that need a second principal or a restart. */ +const GRADEABLE = [ + 'sep-9999-subscribe-webhook-only', + 'sep-9999-subscribe-secret-required', + 'sep-9999-subscribe-secret-format', + 'sep-9999-subscribe-secret-rejected', + 'sep-9999-subscribe-url-https-required', + 'sep-9999-subscribe-url-non-https-rejected', + 'sep-9999-subscribe-key-composition', + 'sep-9999-subscribe-key-immutable', + 'sep-9999-subscribe-idempotent-upsert', + 'sep-9999-subscribe-id-derived', + 'sep-9999-subscribe-id-not-an-input', + 'sep-9999-subscribe-response-cursor', + 'sep-9999-subscribe-response-truncated', + 'sep-9999-ttl-omitted-means-default', + 'sep-9999-ttl-null-only-when-requested', + 'sep-9999-ttl-refresh-before-lte-suggestion', + 'sep-9999-ttl-no-rejection-path', + 'sep-9999-unsubscribe-by-key', + 'sep-9999-unsubscribe-unknown-not-found', + 'sep-9999-error-unsupported' +] as const; + +/** Rows no single-principal, single-process run can exercise. */ +const ALWAYS_UNTESTABLE = [ + 'sep-9999-subscribe-auth-required', + 'sep-9999-subscribe-cross-tenant-isolation', + 'sep-9999-subscribe-refresh-replaces-secret', + 'sep-9999-subscribe-refresh-reactivates', + 'sep-9999-ttl-long-grant-retained', + 'sep-9999-ttl-no-expiry-persisted', + 'sep-9999-ttl-no-expiry-gc-terminated' +] as const; + +async function webhookChecks(opts: EventsFixtureOptions) { + const fixture = await startEventsFixture(opts); + try { + const checks = await new EventsWebhookScenario().run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); + // Drained so an intentionally malformed response does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + takeWireViolations(); + return { + checks: new Map(checks.map((c) => [c.id, c])), + /** What the fixture still holds, which is how cleanup is graded. */ + leaked: fixture.liveSubscriptions() + }; + } finally { + await fixture.close(); + } +} + +/** + * A catalog with one webhook type and one that only polls. The second is not + * decoration: `sep-9999-error-unsupported` needs a type whose `delivery` omits + * webhook, and without one it reports untestable. + */ +function webhookFixture( + subscribe: SubscribeBehaviour = {} +): EventsFixtureOptions { + return { + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }), + descriptor({ name: 'poll.only', delivery: ['poll'] }) + ], + subscribe + }; +} + +describe('the conformant baseline', () => { + test('every gradeable row passes and the untestable ones never read green', async () => { + const { checks, leaked } = await webhookChecks(webhookFixture()); + expect(checks.size).toBe(ALL_ROWS); + for (const id of GRADEABLE) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + for (const id of ALWAYS_UNTESTABLE) { + expect(checks.get(id)?.details?.untestable, id).toBe(true); + expect(checks.get(id)?.status, id).not.toBe('SKIPPED'); + } + // The scenario promises to leave nothing behind, including on a public + // server. This is the only place that promise is actually checked. + expect(leaked).toEqual([]); + }); + + // kitchen-sink allows two subscriptions per principal per event type, and a + // scenario that held every probe open would grade -32013 instead of the rule + // it was probing. The release-as-you-go discipline is what prevents that, and + // this is the regression test for it. + test('a two-subscription cap changes nothing, because probes are released', async () => { + const { checks } = await webhookChecks( + webhookFixture({ maxSubscriptions: 2 }) + ); + for (const id of GRADEABLE) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + }); + + test('a cap of one reports the rows it starves as untestable, not passed', async () => { + const { checks } = await webhookChecks( + webhookFixture({ maxSubscriptions: 1 }) + ); + const check = checks.get('sep-9999-subscribe-key-composition'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('-32013'); + }); + + test('a server with no webhook-capable type reports every row untestable', async () => { + const { checks } = await webhookChecks({ + capability: { listChanged: true }, + descriptors: [descriptor()] + }); + expect(checks.size).toBe(ALL_ROWS); + for (const check of checks.values()) { + expect(check.details?.untestable, check.id).toBe(true); + } + }); + + test('a server that declares nothing and serves nothing skips the suite', async () => { + const { checks } = await webhookChecks({ + listError: { code: -32601, message: 'Method not found' } + }); + for (const check of checks.values()) { + expect(check.status).toBe('SKIPPED'); + } + }); + + test('an unimplemented events/subscribe fails rather than warns', async () => { + const { checks } = await webhookChecks( + webhookFixture({ + error: { code: -32601, message: 'Method not found' } + }) + ); + const check = checks.get('sep-9999-subscribe-webhook-only'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('advertises `webhook` delivery'); + }); + + test('a refused subscribe warns and takes the rest with it as untestable', async () => { + const { checks } = await webhookChecks( + webhookFixture({ error: { code: -32012, message: 'Forbidden' } }) + ); + expect(checks.get('sep-9999-subscribe-webhook-only')?.status).toBe( + 'WARNING' + ); + expect(checks.get('sep-9999-unsubscribe-by-key')?.details?.untestable).toBe( + true + ); + }); +}); + +describe('the delivery secret', () => { + test('accepting a subscribe with no secret fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptMissingSecret: true }) + ); + const check = checks.get('sep-9999-subscribe-secret-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32602 InvalidParams'); + }); + + test('accepting a secret without the whsec_ prefix fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptBadPrefix: true }) + ); + expect(checks.get('sep-9999-subscribe-secret-format')?.status).toBe( + 'FAILURE' + ); + }); + + test('accepting a secret under the 24-byte floor fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptShortSecret: true }) + ); + const check = checks.get('sep-9999-subscribe-secret-rejected'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('accepted'); + }); + + test('rejecting with the wrong code warns and names -32602', async () => { + const { checks } = await webhookChecks( + webhookFixture({ rejectionCode: -32602 + 1 }) + ); + const check = checks.get('sep-9999-subscribe-secret-required'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('-32602 InvalidParams'); + }); +}); + +describe('the callback URL', () => { + // The divergence this row exists for: kitchen-sink accepts `http://` at + // subscribe, deliberately, for demo ergonomics. + test('accepting an http:// callback fails both URL rows off one probe', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptHttpUrl: true }) + ); + const rejected = checks.get('sep-9999-subscribe-url-non-https-rejected'); + expect(rejected?.status).toBe('FAILURE'); + expect(rejected?.errorMessage).toContain('`http://`'); + // The requirement and its enforcement are one probe by design, so the + // second row carries the first's verdict rather than a second probe's. + const required = checks.get('sep-9999-subscribe-url-https-required'); + expect(required?.status).toBe('FAILURE'); + expect(required?.details?.gradedBy).toBe( + 'sep-9999-subscribe-url-non-https-rejected' + ); + }); +}); + +describe('TTL negotiation', () => { + test('no-expiry granted against a finite suggestion fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ nullRefreshAlways: true }) + ); + const check = checks.get('sep-9999-ttl-null-only-when-requested'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('no expiry'); + // With no finite grant there is nothing to compare to the suggestion. + expect( + checks.get('sep-9999-ttl-refresh-before-lte-suggestion')?.details + ?.untestable + ).toBe(true); + }); + + test('no-expiry for an omitted ttlMs fails the default rule', async () => { + const { checks } = await webhookChecks( + webhookFixture({ nullRefreshOnOmitted: true }) + ); + const check = checks.get('sep-9999-ttl-omitted-means-default'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('explicit `ttlMs: null`'); + }); + + test('granting past the suggestion warns', async () => { + const { checks } = await webhookChecks( + webhookFixture({ grantBeyondSuggestion: true }) + ); + const check = checks.get('sep-9999-ttl-refresh-before-lte-suggestion'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('past the suggested'); + }); + + test('a refreshBefore that is not a timestamp fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ refreshBefore: 'next tuesday' }) + ); + const check = checks.get('sep-9999-ttl-refresh-before-lte-suggestion'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('ISO 8601'); + }); + + test('rejecting a ttlMs instead of clamping it fails, and names which', async () => { + const { checks } = await webhookChecks( + webhookFixture({ rejectTtl: 'long' }) + ); + const check = checks.get('sep-9999-ttl-no-rejection-path'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('30 days'); + expect(check?.errorMessage).not.toContain('1000ms'); + }); +}); + +describe('the subscription key and its id', () => { + test('a fresh id per call fails the idempotent upsert', async () => { + const { checks } = await webhookChecks( + webhookFixture({ nonIdempotentId: true }) + ); + const check = checks.get('sep-9999-subscribe-idempotent-upsert'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('created a second subscription'); + }); + + test('an id that ignores delivery.url fails composition and immutability', async () => { + const { checks } = await webhookChecks( + webhookFixture({ idIgnoresUrl: true }) + ); + const composition = checks.get('sep-9999-subscribe-key-composition'); + expect(composition?.status).toBe('FAILURE'); + expect(composition?.errorMessage).toContain('not part of the key'); + // Same probe, so the immutability row moves with it. + expect(checks.get('sep-9999-subscribe-key-immutable')?.status).toBe( + 'FAILURE' + ); + }); + + test('honouring a caller-supplied id fails id-not-an-input', async () => { + const { checks } = await webhookChecks( + webhookFixture({ idIsAnInput: true }) + ); + const check = checks.get('sep-9999-subscribe-id-not-an-input'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('addressed the subscription'); + }); + + test('omitting id fails its own row, and costs the comparison rows too', async () => { + const { checks } = await webhookChecks(webhookFixture({ omitId: true })); + const check = checks.get('sep-9999-subscribe-id-derived'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('expected a string'); + // Not an independent defect: with no id there is nothing to compare, so + // the key rows fail alongside it rather than reporting green. + expect(checks.get('sep-9999-subscribe-key-composition')?.status).toBe( + 'FAILURE' + ); + }); + + test('a numeric cursor on the subscribe response fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ result: { cursor: 42 } }) + ); + expect(checks.get('sep-9999-subscribe-response-cursor')?.status).toBe( + 'FAILURE' + ); + }); + + test('a string truncated on the subscribe response fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ result: { truncated: 'no' } }) + ); + expect(checks.get('sep-9999-subscribe-response-truncated')?.status).toBe( + 'FAILURE' + ); + }); +}); + +describe('unsubscribe', () => { + test('refusing to tear down a key the server holds fails', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeHeldCode: -32603 }) + ); + const check = checks.get('sep-9999-unsubscribe-by-key'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32603'); + // The unknown-key rule is a separate probe and still passes. + expect(checks.get('sep-9999-unsubscribe-unknown-not-found')?.status).toBe( + 'SUCCESS' + ); + }); + + // The divergence this row exists for: kitchen-sink answers success for a key + // it never held, so a client cannot tell teardown from a typo. + test('success for a key never held fails and says why it matters', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeUnknownOk: true }) + ); + const check = checks.get('sep-9999-unsubscribe-unknown-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('typo'); + }); + + test('the wrong code for an unknown key warns and names -32011', async () => { + const { checks } = await webhookChecks( + webhookFixture({ unsubscribeUnknownCode: -32602 }) + ); + const check = checks.get('sep-9999-unsubscribe-unknown-not-found'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('-32011 NotFound'); + }); +}); + +describe('the unsupported delivery mode', () => { + test('subscribing a poll-only type fails the unsupported row', async () => { + const { checks } = await webhookChecks( + webhookFixture({ acceptNonWebhookType: true }) + ); + const check = checks.get('sep-9999-error-unsupported'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('does not advertise'); + }); + + test('a catalog where every type offers webhook reports it untestable', async () => { + const { checks } = await webhookChecks({ + capability: { listChanged: true }, + descriptors: [descriptor({ name: 'hook.event', delivery: ['webhook'] })] + }); + const check = checks.get('sep-9999-error-unsupported'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('no type to probe'); + }); +}); diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts index 30d0622c..86fa918b 100644 --- a/src/scenarios/server/events/negative.test.ts +++ b/src/scenarios/server/events/negative.test.ts @@ -1,12 +1,16 @@ import { describe, test, expect } from 'vitest'; -import { createServer, type IncomingMessage, type Server } from 'http'; -import type { AddressInfo } from 'net'; import { testContext } from '../../../connection/testing'; import { DRAFT_PROTOCOL_VERSION } from '../../../types'; -import { withRequiredDraftResultFields } from '../../../mock-server'; import { takeWireViolations } from '../../../validation/wire-schema'; import { EventsDiscoveryScenario } from './discovery'; import { EventsPollScenario } from './poll'; +import { + descriptor, + occurrence, + pollResult, + startEventsFixture, + type EventsFixtureOptions +} from './negative-fixture'; /** * Negative controls for the MCP Events scenarios. @@ -23,211 +27,26 @@ import { EventsPollScenario } from './poll'; * removal (gap G30). * * The fixture is a minimal SEP-2575 stateless server built per test rather - * than a checked-in example file, matching the SEP-2640 negative tests. An - * events-capable example server is a larger piece of work and belongs with the - * push and webhook scenarios, which genuinely need one. + * than a checked-in example file, matching the SEP-2640 negative tests. It + * lives in negative-fixture.ts, shared with the controls for the other three + * scenarios: negative-push.test.ts, negative-webhook.test.ts and + * negative-delivery.test.ts. */ -/** A descriptor that is well formed apart from whatever a test overrides. */ -function descriptor(overrides: Record = {}) { - return { - name: 'test.event', - description: 'A negative-control fixture event type.', - delivery: ['poll'], - inputSchema: { - type: 'object', - properties: { channel: { type: 'string' } } - }, - payloadSchema: { type: 'object', properties: { id: { type: 'string' } } }, - ...overrides - }; -} - -/** A poll result that is well formed apart from whatever a test overrides. */ -function pollResult(overrides: Record = {}) { - return { - events: [], - cursor: 'cursor_001', - truncated: false, - hasMore: false, - nextPollMs: 30000, - ...overrides - }; -} - -/** An occurrence that is well formed apart from whatever a test overrides. */ -function occurrence(overrides: Record = {}) { - return { - eventId: 'evt_001', - name: 'test.event', - timestamp: '2026-09-15T12:00:00Z', - data: { id: 'x' }, - ...overrides - }; -} - -interface FixtureOptions { - /** Raw value to declare at `capabilities.events`; omit for no declaration. */ - capability?: unknown; - descriptors?: object[]; - /** Answer `events/list` with this JSON-RPC error instead of a result. */ - listError?: { code: number; message: string }; - /** - * Poll responses, consumed in order; the last one repeats once exhausted. - * A `{ error }` entry makes that poll answer with a JSON-RPC error. - */ - pollResponses?: Array< - Record | { error: { code: number; message: string } } - >; - /** Overrides keyed by the polled event name, taking priority over the queue. */ - pollByName?: Record< - string, - Record | { error: { code: number; message: string } } - >; - /** Error code for a poll naming an event type the fixture does not serve. */ - unknownNameCode?: number; - /** Error code for a poll whose arguments violate `inputSchema`. */ - invalidArgsCode?: number; -} - -function startFixture(opts: FixtureOptions): Promise<{ - url: string; - server: Server; - polls: Array>; -}> { - const polls: Array> = []; - const queue = [...(opts.pollResponses ?? [pollResult()])]; - const descriptors = opts.descriptors ?? [descriptor()]; - const names = new Set( - descriptors - .map((d) => (d as { name?: unknown }).name) - .filter((n): n is string => typeof n === 'string') - ); - - const server = createServer(async (req, res) => { - if (req.method !== 'POST') { - res.writeHead(405).end(); - return; - } - const body = await readJsonBody(req); - const method = body.method as string; - const id = body.id; - const params = (body.params ?? {}) as Record; - - const send = (result: object) => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end( - JSON.stringify({ - jsonrpc: '2.0', - id, - result: withRequiredDraftResultFields(method, result) - }) - ); - }; - const fail = (code: number, message: string, data?: unknown) => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end( - JSON.stringify({ jsonrpc: '2.0', id, error: { code, message, data } }) - ); - }; - - if (method === 'server/discover') { - send({ - supportedVersions: [DRAFT_PROTOCOL_VERSION], - capabilities: 'capability' in opts ? { events: opts.capability } : {}, - serverInfo: { name: 'events-negative', version: '1.0.0' } - }); - return; - } - - if (method === 'events/list') { - if (opts.listError) { - fail(opts.listError.code, opts.listError.message); - return; - } - send({ events: descriptors }); - return; - } - - if (method === 'events/poll') { - polls.push(params); - const name = params.name; - - if (typeof name !== 'string') { - fail(-32602, 'InvalidParams: `name` is required'); - return; - } - if (!names.has(name)) { - fail(opts.unknownNameCode ?? -32011, 'NotFound', { kind: 'event' }); - return; - } - - const byName = opts.pollByName?.[name]; - const chosen = - byName ?? - (queue.length > 1 ? queue.shift()! : (queue[0] ?? pollResult())); - - // Argument validation against the fixture's own declared schema, so the - // invalid-arguments probe has something real to violate. - const args = (params.arguments ?? {}) as Record; - const decl = descriptors.find( - (d) => (d as { name?: unknown }).name === name - ) as { inputSchema?: { properties?: Record } }; - for (const [key, value] of Object.entries(args)) { - const declared = decl?.inputSchema?.properties?.[key]; - if (declared?.type === 'string' && typeof value !== 'string') { - fail(opts.invalidArgsCode ?? -32602, 'InvalidParams'); - return; - } - } - - if ('error' in chosen) { - const e = (chosen as { error: { code: number; message: string } }) - .error; - fail(e.code, e.message); - return; - } - send(chosen as Record); - return; - } - - fail(-32601, `Method not found: ${method}`); - }); - - return new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, () => { - const addr = server.address() as AddressInfo; - resolve({ url: `http://localhost:${addr.port}/mcp`, server, polls }); - }); - }); -} - -async function readJsonBody( - req: IncomingMessage -): Promise> { - const chunks: Buffer[] = []; - for await (const chunk of req) { - chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); - } - return JSON.parse(Buffer.concat(chunks).toString('utf8')) as Record< - string, - unknown - >; -} - type Scenario = EventsDiscoveryScenario | EventsPollScenario; -async function checksFor(scenario: Scenario, opts: FixtureOptions) { - const { url, server } = await startFixture(opts); +async function checksFor(scenario: Scenario, opts: EventsFixtureOptions) { + const fixture = await startEventsFixture(opts); try { - const checks = await scenario.run(testContext(url, DRAFT_PROTOCOL_VERSION)); + const checks = await scenario.run( + testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + ); // Drained so an intentionally malformed response does not trip the global // vitest hook; these tests assert on the check, not the wire validator. takeWireViolations(); return new Map(checks.map((c) => [c.id, c])); } finally { - await new Promise((r) => server.close(() => r())); + await fixture.close(); } } @@ -235,7 +54,7 @@ const discovery = () => new EventsDiscoveryScenario(); const poll = () => new EventsPollScenario(); /** The baseline every negative case is compared against. */ -const CONFORMANT: FixtureOptions = { +const CONFORMANT: EventsFixtureOptions = { capability: { listChanged: true }, descriptors: [descriptor()], pollResponses: [pollResult()] diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 59be3088..4aa5e26b 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -114,12 +114,25 @@ # real SSE stream open through stream.ts, because conn.request() resolves # on the response for its id and a push stream withholds that until the # subscription ends. -# webhook.ts (events-webhook), 26 rows — sep-9999-subscribe-*, +# webhook.ts (events-webhook), 27 rows — sep-9999-subscribe-*, # sep-9999-ttl-*, sep-9999-unsubscribe-* and sep-9999-error-unsupported. # webhook-delivery.ts (events-webhook-delivery), 28 rows — # sep-9999-delivery-*, sep-9999-verification-*, sep-9999-ssrf-* and # sep-9999-envelope-*, graded from an HTTP receiver the harness runs. # +# negative controls: src/scenarios/server/events/negative*.test.ts, one file per +# scenario over a shared fixture in negative-fixture.ts. A green run against +# kitchen-sink proves a check does not false-positive; only a fixture broken in +# exactly one way proves it catches anything. Every divergence listed below has +# a case there, so a red run against a real server can be trusted rather than +# re-derived. +# +# The fixture can also do the four things no client can ask a real server for +# (fail upstream, lose its replay window, terminate a subscription, close a +# stream itself) plus the gap and terminated control envelopes. Those rows +# report untestable against both implementations, so the controls are the only +# evidence those checks work at all. +# # The 14 rows nothing emits yet, and what each needs: # sep-9999-schema-evolution-additive, sep-9999-breaking-change-new-name, # sep-9999-list-changed-notification and the four remaining From 0ffd1ceb1111efc59c54921c3cf1ebabb8267ebd Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Tue, 22 Sep 2026 13:46:10 +0000 Subject: [PATCH 10/27] test(events): run the push and delivery controls concurrently MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two files that grade timing spent almost all of their wall time asleep: the delivery controls waited out five settle windows per case and took 180s, the push controls 80s. Both together took the suite from 163s to 425s, which every contributor pays on pre-push. The cases are independent — each builds its own fixture on its own port and touches no shared state — so the only thing keeping them sequential was `vi.resetModules()` per case, needed because both scenarios read their timing knobs when the module is first evaluated. Importing once per file in `beforeAll` removes that, and the cases then run concurrently: delivery 45s, push 43s, suite 254s. One graph means one set of timings, and one push case has to outlast the 30s heartbeat cadence the document permits, because under 30s a silent server and a slow one are indistinguishable. It takes its own graph in a sequential block at the foot of the file, after the concurrent cases are done with theirs; resetting the registry while they were still running would pull it out from under them. 725 passing, unchanged. --- .../server/events/negative-delivery.test.ts | 75 +++++++---- .../server/events/negative-push.test.ts | 119 +++++++++++------- 2 files changed, 127 insertions(+), 67 deletions(-) diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index 93965c4e..6c9be756 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect, vi, afterEach } from 'vitest'; +import { describe, test, expect, vi, beforeAll, afterAll } from 'vitest'; import { DRAFT_PROTOCOL_VERSION } from '../../../types'; import { descriptor, @@ -25,13 +25,27 @@ import { * Timings are stubbed down hard. `EVENTS_DELIVERY_WAIT_MS` bounds how long the * scenario waits for a first POST and `EVENTS_DELIVERY_SETTLE_MS` how long it * lets retries play out; at their defaults one run of this scenario is about - * twenty seconds of mostly sleeping. As in the push controls, the run context - * has to come from the same freshly-imported module graph as the scenario, or - * `instanceof JsonRpcError` fails across two copies of the class. + * twenty seconds of mostly sleeping, and even stubbed down it is about ten. + * + * So the cases run concurrently. Each builds its own fixture on its own port and + * touches no shared state, and the scenario is imported once in `beforeAll` + * rather than per case, which is what makes that safe: `vi.resetModules()` in + * the middle of a concurrent run would pull the module graph out from under a + * case already using it. One graph also means one set of timings for the file. + * + * As in the push controls, the run context has to come from the same fresh + * import as the scenario, or `instanceof JsonRpcError` is false across two + * copies of the class. */ -async function deliveryChecks(opts: EventsFixtureOptions) { - vi.resetModules(); +type Imported = { + Scenario: typeof import('./webhook-delivery').EventsWebhookDeliveryScenario; + testContext: typeof import('../../../connection/testing').testContext; + takeWireViolations: typeof import('../../../validation/wire-schema').takeWireViolations; +}; +let mod: Imported; + +beforeAll(async () => { vi.stubEnv('EVENTS_DELIVERY_WAIT_MS', '1500'); // Comfortably over the fixture's 1.1s retry spacing, which is itself over a // second because `webhook-timestamp` is in whole seconds and two attempts @@ -40,16 +54,32 @@ async function deliveryChecks(opts: EventsFixtureOptions) { // Never inherit a real tunnel from the environment: these tests are about the // loopback receiver, which is also the SSRF probe. vi.stubEnv('EVENTS_WEBHOOK_CALLBACK_BASE', ''); - const { EventsWebhookDeliveryScenario } = await import('./webhook-delivery'); - const { testContext } = await import('../../../connection/testing'); - const { takeWireViolations } = - await import('../../../validation/wire-schema'); + vi.resetModules(); + const [scenario, testing, wire] = await Promise.all([ + import('./webhook-delivery'), + import('../../../connection/testing'), + import('../../../validation/wire-schema') + ]); + mod = { + Scenario: scenario.EventsWebhookDeliveryScenario, + testContext: testing.testContext, + takeWireViolations: wire.takeWireViolations + }; +}); + +afterAll(() => { + vi.unstubAllEnvs(); +}); + +async function deliveryChecks(opts: EventsFixtureOptions) { const fixture = await startEventsFixture(opts); try { - const checks = await new EventsWebhookDeliveryScenario().run( - testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + const checks = await new mod.Scenario().run( + mod.testContext(fixture.url, DRAFT_PROTOCOL_VERSION) ); - takeWireViolations(); + // Drained so an intentionally malformed delivery does not trip the global + // vitest hook; these tests assert on the check, not the wire validator. + mod.takeWireViolations(); return new Map(checks.map((c) => [c.id, c])); } finally { await fixture.close(); @@ -71,13 +101,10 @@ function delivering(delivery: DeliveryBehaviour = {}): EventsFixtureOptions { }; } -const TIMEOUT = 40_000; - -afterEach(() => { - vi.unstubAllEnvs(); -}); +/** Generous, because every case in the file is in flight at once. */ +const TIMEOUT = 60_000; -describe('a server that delivers to loopback', () => { +describe.concurrent('a server that delivers to loopback', () => { test( 'every row but the SSRF pair grades, and the SSRF pair fails', async () => { @@ -162,7 +189,7 @@ describe('a server that delivers to loopback', () => { ); }); -describe('the verification handshake', () => { +describe.concurrent('the verification handshake', () => { // The divergence this row exists for: kitchen-sink delivers with no handshake // at all, because the handshake does not exist yet. test( @@ -197,7 +224,7 @@ describe('the verification handshake', () => { ); }); -describe('signing', () => { +describe.concurrent('signing', () => { // The divergence this row exists for: kitchen-sink keys the HMAC on the // literal `whsec_…` string, where the document says the key is the // base64-decoded bytes after the prefix. No Standard Webhooks receiver @@ -263,7 +290,7 @@ describe('signing', () => { ); }); -describe('transport and body', () => { +describe.concurrent('transport and body', () => { test( 'delivering as text/plain fails the POST-and-JSON row', async () => { @@ -289,7 +316,7 @@ describe('transport and body', () => { ); }); -describe('retries and redirects', () => { +describe.concurrent('retries and redirects', () => { test( 'reusing the timestamp across retries fails the freshness row', async () => { @@ -333,7 +360,7 @@ describe('retries and redirects', () => { ); }); -describe('control envelopes', () => { +describe.concurrent('control envelopes', () => { test( 'an unsigned envelope fails, since a receiver cannot verify it', async () => { diff --git a/src/scenarios/server/events/negative-push.test.ts b/src/scenarios/server/events/negative-push.test.ts index 76d87958..0bf65039 100644 --- a/src/scenarios/server/events/negative-push.test.ts +++ b/src/scenarios/server/events/negative-push.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect, vi, afterEach } from 'vitest'; +import { describe, test, expect, vi, beforeAll, afterAll } from 'vitest'; import { DRAFT_PROTOCOL_VERSION } from '../../../types'; import { descriptor, @@ -21,37 +21,68 @@ import { * they are graded here rather than only declared — which is the only evidence * that those checks work at all. * - * `EVENTS_PUSH_WATCH_MS` is stubbed down from 35s per case, since the scenario - * reads it when the module is first evaluated. `vi.resetModules()` before each - * dynamic import is what makes the stub land regardless of whether another test - * file imported the scenario first. The catch is that a reset registry hands - * back a *second* copy of the connection module, and `err instanceof - * JsonRpcError` is false across two copies of the same class — so the run - * context has to be built from the same fresh graph as the scenario, not from a - * static import up here. + * `EVENTS_PUSH_WATCH_MS` is stubbed down from 35s, since the scenario reads it + * when the module is first evaluated, and `vi.resetModules()` before the dynamic + * import is what makes the stub land regardless of whether another test file + * imported the scenario first. Two catches follow from that. A reset registry + * hands back a second copy of the connection module, and `err instanceof + * JsonRpcError` is false across two copies of the same class, so the run context + * has to come from the same fresh graph as the scenario rather than a static + * import up here. And the window is fixed at import, so one graph means one + * window. + * + * The fast cases share one 900ms graph, imported once, and run concurrently: + * each builds its own fixture on its own port and touches nothing shared. The + * single case that has to outlast the 30s cadence the document permits takes its + * own graph, sequentially, at the foot of the file — a `resetModules()` while + * the concurrent cases were still running would pull their graph out from under + * them. */ /** The window every case uses, except the one that needs to outlast 30s. */ const FAST_WATCH_MS = 900; -async function pushChecks( - opts: EventsFixtureOptions, - watchMs: number = FAST_WATCH_MS -) { - vi.resetModules(); +type Imported = { + Scenario: typeof import('./push').EventsPushScenario; + testContext: typeof import('../../../connection/testing').testContext; + takeWireViolations: typeof import('../../../validation/wire-schema').takeWireViolations; +}; + +/** The scenario, with one watch window, in its own module registry. */ +async function importWith(watchMs: number): Promise { vi.stubEnv('EVENTS_PUSH_WATCH_MS', String(watchMs)); - const { EventsPushScenario } = await import('./push'); - const { testContext } = await import('../../../connection/testing'); - const { takeWireViolations } = - await import('../../../validation/wire-schema'); + vi.resetModules(); + const [scenario, testing, wire] = await Promise.all([ + import('./push'), + import('../../../connection/testing'), + import('../../../validation/wire-schema') + ]); + return { + Scenario: scenario.EventsPushScenario, + testContext: testing.testContext, + takeWireViolations: wire.takeWireViolations + }; +} + +let fast: Imported; + +beforeAll(async () => { + fast = await importWith(FAST_WATCH_MS); +}); + +afterAll(() => { + vi.unstubAllEnvs(); +}); + +async function pushChecks(opts: EventsFixtureOptions, using: Imported = fast) { const fixture = await startEventsFixture(opts); try { - const checks = await new EventsPushScenario().run( - testContext(fixture.url, DRAFT_PROTOCOL_VERSION) + const checks = await new using.Scenario().run( + using.testContext(fixture.url, DRAFT_PROTOCOL_VERSION) ); // Drained so an intentionally malformed frame does not trip the global // vitest hook; these tests assert on the check, not the wire validator. - takeWireViolations(); + using.takeWireViolations(); return new Map(checks.map((c) => [c.id, c])); } finally { await fixture.close(); @@ -67,11 +98,7 @@ function pushFixture(stream: StreamBehaviour = {}): EventsFixtureOptions { }; } -afterEach(() => { - vi.unstubAllEnvs(); -}); - -describe('events/stream opening', () => { +describe.concurrent('events/stream opening', () => { test('the conformant fixture passes every gradeable row', async () => { const checks = await pushChecks(pushFixture()); for (const id of [ @@ -153,7 +180,7 @@ describe('events/stream opening', () => { }); }); -describe('the active confirmation', () => { +describe.concurrent('the active confirmation', () => { test('no confirmation at all fails', async () => { const checks = await pushChecks(pushFixture({ omitActive: true })); const check = checks.get('sep-9999-stream-active-confirmation'); @@ -180,7 +207,7 @@ describe('the active confirmation', () => { }); }); -describe('the correlation id', () => { +describe.concurrent('the correlation id', () => { // The divergence this row exists for: mcpkit puts the id in params.requestId, // mirroring the sketch's own push examples, where the document requires the // SEP-2575 `_meta` spelling. A client holding two streams cannot route by @@ -202,7 +229,7 @@ describe('the correlation id', () => { }); }); -describe('what rides the stream', () => { +describe.concurrent('what rides the stream', () => { test('a non-events notification fails and names the method', async () => { const checks = await pushChecks( pushFixture({ foreignNotification: 'notifications/message' }) @@ -231,7 +258,7 @@ describe('what rides the stream', () => { }); }); -describe('the heartbeat', () => { +describe.concurrent('the heartbeat', () => { test('a numeric cursor on the heartbeat fails', async () => { const checks = await pushChecks( pushFixture({ heartbeatParams: { cursor: 42 } }) @@ -249,17 +276,6 @@ describe('the heartbeat', () => { expect(check?.errorMessage).toContain('EVENTS_PUSH_WATCH_MS'); }); - // The one case that has to outlast the cadence the document permits. Under - // 30s a silent server and a slow one are indistinguishable, so this is the - // only window in which the MUST can actually fail. - test('silence past 30s fails the heartbeat MUST', async () => { - const checks = await pushChecks(pushFixture({ heartbeatMs: 0 }), 30_050); - const check = checks.get('sep-9999-stream-heartbeat-required'); - expect(check?.status).toBe('FAILURE'); - expect(check?.details?.untestable).toBeUndefined(); - expect(check?.errorMessage).toContain('outlasts the 30s cadence'); - }, 60_000); - test('an SSE comment keepalive beside a data heartbeat warns', async () => { const checks = await pushChecks(pushFixture({ sseComments: true })); const check = checks.get('sep-9999-stream-heartbeat-not-sse-comment'); @@ -277,7 +293,7 @@ describe('the heartbeat', () => { }); }); -describe('concurrency and cancellation', () => { +describe.concurrent('concurrency and cancellation', () => { // The cap kitchen-sink applies to streams, which the document exempts them // from: the first stream confirms and the other two are refused -32013. test('a per-principal cap that catches streams fails', async () => { @@ -298,7 +314,7 @@ describe('concurrency and cancellation', () => { }); }); -describe('rows that are untestable against a real server', () => { +describe.concurrent('rows that are untestable against a real server', () => { test('a server-initiated close grades the final result instead of skipping it', async () => { const cancelled = await pushChecks(pushFixture()); const untested = cancelled.get('sep-9999-stream-final-result-shape'); @@ -349,3 +365,20 @@ describe('rows that are untestable against a real server', () => { } }); }); + +/** + * Under 30s a silent server and a slow one are indistinguishable, so this is the + * only window in which the heartbeat MUST can fail rather than report a window + * too short to judge. It costs half a minute, which is why it is one case and + * not a pair, and why it runs last on a graph of its own. + */ +describe('the heartbeat MUST, on a window that outlasts the cadence', () => { + test('silence past 30s fails instead of reporting untestable', async () => { + const slow = await importWith(30_050); + const checks = await pushChecks(pushFixture({ heartbeatMs: 0 }), slow); + const check = checks.get('sep-9999-stream-heartbeat-required'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBeUndefined(); + expect(check?.errorMessage).toContain('outlasts the 30s cadence'); + }, 90_000); +}); From d3d26e0e657f28d79bc47e82c6546958f10e6550 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Wed, 23 Sep 2026 17:21:00 +0000 Subject: [PATCH 11/27] fix(events): the capability declares through the extensions map The design sketch put it top-level under `capabilities.events`. metronome, the sketch author's own server, used `capabilities.extensions`. Raised with him 2026-09-22 rather than resolved by picking a side, and the document turned out to be the thing that was wrong: upstream PR 7 moves the sketch into Extension Negotiation, keyed by `io.modelcontextprotocol/events`. `declaredEventsCapability` and the four scenarios that probe the declaration now read the extensions map. `EVENTS_CAPABILITY` is gone; `EVENTS_EXTENSION_ID` absorbed its job, having previously carried a comment saying in as many words not to read the capability at that key. Two rows are restated from PR 7 and four are new: the empty settings object, the optional client declaration, `listChanged` gating whether the server may send `notifications/events/list_changed`, and the `-32601` fallback for a server that does not offer the extension. The four are declared and not yet emitted, so they report untested. The header's record of both metronome divergences becomes a record of how they resolved. He fixed the `-32603`-instead-of-`-32602` one the same day. These rows track an unmerged PR, which the head of this file now says. Refs #504 --- src/scenarios/server/events/discovery.ts | 11 ++-- src/scenarios/server/events/helpers.ts | 49 ++++++++++-------- .../server/events/negative-fixture.ts | 11 +++- src/scenarios/server/events/negative.test.ts | 4 +- src/scenarios/server/events/push.ts | 6 +-- .../server/events/webhook-delivery.ts | 4 +- src/scenarios/server/events/webhook.ts | 4 +- src/seps/sep-9999.yaml | 50 +++++++++++-------- 8 files changed, 82 insertions(+), 57 deletions(-) diff --git a/src/scenarios/server/events/discovery.ts b/src/scenarios/server/events/discovery.ts index 16cb0a15..6829d8f0 100644 --- a/src/scenarios/server/events/discovery.ts +++ b/src/scenarios/server/events/discovery.ts @@ -31,7 +31,6 @@ import type { RunContext } from '../../../connection'; import { untestableCheck } from '../../untestable'; import { EVENTS_EXTENSION_ID, - EVENTS_CAPABILITY, EVENTS_LIST_METHOD, EVENTS_POLL_METHOD, EVENTS_SPEC_REF, @@ -110,7 +109,7 @@ export class EventsDiscoveryScenario implements ClientScenario { - \`sep-9999-error-not-found\` — an unknown event name answers \`-32011 NotFound\` (the poll-specific restatement of the same rule is graded by \`events-poll\`) - \`sep-9999-error-server-range\` — the extension's codes sit in the JSON-RPC implementation-defined server range -**Discovery is dynamic**: a server that neither declares the capability nor implements \`events/list\` SKIPs everything. One that answers \`events/list\` without declaring the capability is graded, and fails the declaration check, because that surface is unreachable for a client that reads capabilities first. An empty catalog reports the descriptor checks as untestable rather than passing them.`; +**Discovery is dynamic**: a server that neither declares the extension nor implements \`events/list\` SKIPs everything. One that answers \`events/list\` without declaring the extension is graded, and fails the declaration check, because that surface is unreachable for a client that reads capabilities first. An empty catalog reports the descriptor checks as untestable rather than passing them.`; async run(ctx: RunContext): Promise { const conn = await ctx.connect(); @@ -148,8 +147,8 @@ export class EventsDiscoveryScenario implements ClientScenario { capDescription, 'FAILURE', { - errorMessage: `Server answers \`${EVENTS_LIST_METHOD}\` but declares no \`capabilities.${EVENTS_CAPABILITY}\`. A client that follows the spec reads capabilities to decide whether to call it, so this surface is unreachable.`, - details: { capabilities: EVENTS_CAPABILITY, declared: false } + errorMessage: `Server answers \`${EVENTS_LIST_METHOD}\` but declares no \`capabilities.extensions["${EVENTS_EXTENSION_ID}"]\`. A client that follows the spec reads capabilities to decide whether to call it, so this surface is unreachable.`, + details: { extensionId: EVENTS_EXTENSION_ID, declared: false } } ) ); @@ -168,7 +167,7 @@ export class EventsDiscoveryScenario implements ClientScenario { capDescription, 'FAILURE', { - errorMessage: `\`capabilities.${EVENTS_CAPABILITY}\` is ${describeValue(value)}, expected an object.`, + errorMessage: `\`capabilities.extensions["${EVENTS_EXTENSION_ID}"]\` is ${describeValue(value)}, expected an object.`, details: { declared: value } } ) @@ -225,7 +224,7 @@ export class EventsDiscoveryScenario implements ClientScenario { 'FAILURE', { errorMessage: unimplemented - ? `Server declares \`capabilities.events\` but \`${EVENTS_LIST_METHOD}\` is not implemented (-32601).` + ? `Server declares \`capabilities.extensions["${EVENTS_EXTENSION_ID}"]\` but \`${EVENTS_LIST_METHOD}\` is not implemented (-32601).` : `\`${EVENTS_LIST_METHOD}\` failed: ${err.code} ${err.message}`, details: { code: err.code, message: err.message, data: err.data } } diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 6b39ca38..f5e0c8b2 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -8,16 +8,10 @@ * src/seps/sep-9999.yaml, and 9999 is a placeholder SEP number — see that * file's header before renaming anything here. * - * Two things about Events differ from every other extension suite here and are + * One thing about Events differs from every other extension suite here and is * worth knowing before reading the scenarios: * - * 1. The capability is declared at the top level as `capabilities.events`, not - * under `capabilities.extensions`. SEP-2133's extensions map does not come - * into it. `EVENTS_EXTENSION_ID` exists only as a `ScenarioSource` key so - * the runner keeps these scenarios off the `--spec-version` timeline; it is - * never a path into the capability object. - * - * 2. No delivery mode is mandatory. A descriptor's `delivery` array is any + * 1. No delivery mode is mandatory. A descriptor's `delivery` array is any * non-empty subset of poll/push/webhook, so a scenario for one mode has to * discover whether any event type offers it before it can probe anything. * Nothing is hardcoded to a fixture's event names. @@ -32,20 +26,21 @@ import type { Connection } from '../../../connection'; import { JsonRpcError } from '../../../connection'; /** - * Suite-selection key for the Events scenarios. + * The Events extension identifier, per SEP-2133 Extension Negotiation. + * + * It does double duty: `ScenarioSource` carries it as `{ extensionId }`, which + * keeps these scenarios out of `--spec-version` selection (see + * `matchesSpecVersion` in src/scenarios/index.ts), and it is the key the + * capability itself lives under in `capabilities.extensions`. * - * Events has no SEP-2133 extension identifier, because it declares its - * capability top-level rather than inside `capabilities.extensions`. This - * string exists so `ScenarioSource` can carry `{ extensionId }`, which is what - * keeps the scenarios out of `--spec-version` selection (see - * `matchesSpecVersion` in src/scenarios/index.ts). Do not read the capability - * at this key; read `capabilities.events`. + * Those were two different things until 2026-09-22. The design sketch put the + * capability at the top level as `capabilities.events`, and this file said in + * as many words not to read the capability at this key. Upstream PR 7 moved the + * sketch into the extensions map after mcpkit followed the document and the + * reference server did not, so the two uses collapsed into one. */ export const EVENTS_EXTENSION_ID = 'io.modelcontextprotocol/events'; -/** The capability key, top-level under `capabilities`. */ -export const EVENTS_CAPABILITY = 'events'; - export const EVENTS_LIST_METHOD = 'events/list'; export const EVENTS_POLL_METHOD = 'events/poll'; export const EVENTS_STREAM_METHOD = 'events/stream'; @@ -186,9 +181,23 @@ export async function declaredEventsCapability( ): Promise<{ declared: boolean; value: unknown }> { const discovered = await conn.discover(); const caps = (discovered.capabilities as Record) ?? {}; - if (!(EVENTS_CAPABILITY in caps)) + const exts = (caps.extensions as Record) ?? {}; + if (!(EVENTS_EXTENSION_ID in exts)) return { declared: false, value: undefined }; - return { declared: true, value: caps[EVENTS_CAPABILITY] }; + return { declared: true, value: exts[EVENTS_EXTENSION_ID] }; +} + +/** + * The `extensions` map from a capabilities object, or an empty map. + * + * Every scenario that probes the declaration goes through this rather than + * indexing `capabilities` directly, so the one place that knows where the + * capability lives is this file. + */ +export function extensionsOf( + caps: Record +): Record { + return (caps.extensions as Record) ?? {}; } /** diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index fd349f24..6b995cd7 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -13,6 +13,7 @@ * cadence and the cancellation rows are about timing. */ +import { EVENTS_EXTENSION_ID } from './helpers'; import { createHash, createHmac } from 'crypto'; import { createServer, type IncomingMessage, type ServerResponse } from 'http'; import type { AddressInfo } from 'net'; @@ -225,7 +226,10 @@ export interface DeliveryBehaviour { } export interface EventsFixtureOptions { - /** Raw value to declare at `capabilities.events`; omit for no declaration. */ + /** + * Raw value to declare at `capabilities.extensions["io.modelcontextprotocol/events"]`; + * omit for no declaration. + */ capability?: unknown; descriptors?: object[]; /** Answer `events/list` with this JSON-RPC error instead of a result. */ @@ -337,7 +341,10 @@ export async function startEventsFixture( if (method === 'server/discover') { send({ supportedVersions: [DRAFT_PROTOCOL_VERSION], - capabilities: 'capability' in opts ? { events: opts.capability } : {}, + capabilities: + 'capability' in opts + ? { extensions: { [EVENTS_EXTENSION_ID]: opts.capability } } + : {}, serverInfo: { name: 'events-negative', version: '1.0.0' } }); return; diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts index 86fa918b..b790e778 100644 --- a/src/scenarios/server/events/negative.test.ts +++ b/src/scenarios/server/events/negative.test.ts @@ -92,7 +92,7 @@ describe('events capability declaration', () => { }); const cap = checks.get('sep-9999-capability-events-object'); expect(cap?.status).toBe('FAILURE'); - expect(cap?.errorMessage).toContain('declares no `capabilities.events`'); + expect(cap?.errorMessage).toContain('declares no `capabilities.extensions'); expect(cap?.errorMessage).toContain('unreachable'); // And the rest of the scenario is still graded, not abandoned. @@ -183,7 +183,7 @@ describe('events/list descriptors', () => { }); const check = checks.get('sep-9999-list-implemented'); expect(check?.status).toBe('FAILURE'); - expect(check?.errorMessage).toContain('declares `capabilities.events`'); + expect(check?.errorMessage).toContain('declares `capabilities.extensions'); }); }); diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts index 222e5418..460576c2 100644 --- a/src/scenarios/server/events/push.ts +++ b/src/scenarios/server/events/push.ts @@ -29,10 +29,10 @@ import type { RunContext } from '../../../connection'; import { untestableCheck } from '../../untestable'; import { EVENTS_ACTIVE_NOTIFICATION, - EVENTS_CAPABILITY, + EVENTS_EXTENSION_ID, + extensionsOf, EVENTS_ERROR_NOTIFICATION, EVENTS_EVENT_NOTIFICATION, - EVENTS_EXTENSION_ID, EVENTS_HEARTBEAT_NOTIFICATION, EVENTS_SPEC_REF, EVENTS_STREAM_METHOD, @@ -125,7 +125,7 @@ export class EventsPushScenario implements ClientScenario { const caps = isObject(capabilities.capabilities) ? capabilities.capabilities : {}; - declared = caps[EVENTS_CAPABILITY] !== undefined; + declared = extensionsOf(caps)[EVENTS_EXTENSION_ID] !== undefined; const listed = await eventsListAll(conn); if ('error' in listed) { diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index 2cba5c80..d74faa11 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -32,9 +32,9 @@ import type { Connection, RunContext } from '../../../connection'; import { JsonRpcError } from '../../../connection'; import { untestableCheck } from '../../untestable'; import { - EVENTS_CAPABILITY, EVENTS_CALLBACK_ENDPOINT_ERROR, EVENTS_EXTENSION_ID, + extensionsOf, EVENTS_SPEC_REF, EVENTS_SUBSCRIBE_METHOD, EVENTS_UNSUBSCRIBE_METHOD, @@ -193,7 +193,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { const caps = isObject(capabilities.capabilities) ? capabilities.capabilities : {}; - const declared = caps[EVENTS_CAPABILITY] !== undefined; + const declared = extensionsOf(caps)[EVENTS_EXTENSION_ID] !== undefined; const listed = await eventsListAll(conn); if ('error' in listed) { diff --git a/src/scenarios/server/events/webhook.ts b/src/scenarios/server/events/webhook.ts index 14d82856..933ce7a7 100644 --- a/src/scenarios/server/events/webhook.ts +++ b/src/scenarios/server/events/webhook.ts @@ -32,8 +32,8 @@ import type { Connection, RunContext } from '../../../connection'; import { JsonRpcError } from '../../../connection'; import { untestableCheck } from '../../untestable'; import { - EVENTS_CAPABILITY, EVENTS_EXTENSION_ID, + extensionsOf, EVENTS_NOT_FOUND, EVENTS_SPEC_REF, EVENTS_SUBSCRIBE_METHOD, @@ -171,7 +171,7 @@ export class EventsWebhookScenario implements ClientScenario { const caps = isObject(capabilities.capabilities) ? capabilities.capabilities : {}; - const declared = caps[EVENTS_CAPABILITY] !== undefined; + const declared = extensionsOf(caps)[EVENTS_EXTENSION_ID] !== undefined; const listed = await eventsListAll(conn); if ('error' in listed) { diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 4aa5e26b..52a99629 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -173,10 +173,12 @@ # mcpkit pass; catching them is the point. # # sep-9999-capability-events-object — the server answers `events/list` but -# declares no `capabilities.events`, so a client that reads capabilities -# before calling never finds the surface. Not previously tracked. This is -# also why the scenario asks before it skips: a plain "undeclared optional -# capability" SKIP reports this server as a clean run. +# declares nothing, so a client that reads capabilities before calling never +# finds the surface. Not previously tracked. This is also why the scenario +# asks before it skips: a plain "undeclared optional capability" SKIP +# reports this server as a clean run. Closed by mcpkit 1416 (declaring +# top-level, which the document then said) and corrected to the extensions +# map by mcpkit 1421 once upstream PR 7 moved it. # sep-9999-poll-next-poll-ms — `nextPollSeconds` persists (events.go:504) # after `197c32b4` renamed it, and `maxAge` is still in seconds # (events.go:544). mcpkit's own wire_shape_test.go:44 asserts the @@ -225,23 +227,23 @@ # Its one event type has required `inputSchema` properties. See the # `minimalArguments` note below. # -# Two divergences, neither previously known: +# Two divergences, neither previously known. Both are now resolved, and the +# way they resolved is the argument for running a suite against more than one +# implementation: # -# sep-9999-capability-events-object — metronome declares events under +# sep-9999-capability-events-object — metronome declared events under # `capabilities.extensions["io.modelcontextprotocol/events"]`, the SEP-2133 -# extensions map, where this document puts it top-level under -# `capabilities`. Ask before changing the row. The document says top-level -# and has not moved since `28ec35e9`, but metronome is the sketch author's -# own server, and every other extension in MCP declares itself through the -# extensions map. If the WG confirms the extensions map, this row and -# `EVENTS_CAPABILITY` in src/scenarios/server/events/helpers.ts both change, -# and mcpkit's failure on the same row means something different than it -# does today. +# extensions map, where this document put it top-level under +# `capabilities`. Raised with the author 2026-09-22 rather than resolved by +# picking a side. **The document was wrong**: extension negotiation is where +# it belongs, and upstream PR 7 moves the sketch there. metronome was right +# all along and mcpkit's conformance was an artifact of following a document +# that did not yet say what its author meant. The rows above now track PR 7, +# which is unmerged; see the tracking note at the head of this file. # sep-9999-poll-invalid-arguments — arguments that violate `inputSchema` -# answer -32603 Internal error, where the document requires -32602 -# InvalidParams. The message ("Got JSON of type string, expected int_") -# reads like a deserialization failure surfacing rather than a validation -# step, so it is likely a framework default rather than a decision. +# answered -32603 Internal error, where the document requires -32602 +# InvalidParams. Read as a framework default surfacing rather than a +# decision, which is what it was. Fixed by the author 2026-09-22. # # `sep-9999-poll-mode-unsupported` reports untestable here, for the opposite # reason it fails against mcpkit: every event type metronome serves offers @@ -382,9 +384,17 @@ spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-even requirements: # === Capability Declaration === - check: sep-9999-capability-events-object - text: 'Servers advertise event support in their capabilities: `{"capabilities": {"events": {"listChanged": true}}}`.' + text: "Events is an MCP extension, identified as `io.modelcontextprotocol/events`, and is declared through Extension Negotiation: the identifier appears as a key in the `extensions` field of capabilities, mapped to the extension's settings object." - check: sep-9999-capability-list-changed-flag - text: 'The `listChanged` flag under `capabilities.events` advertises that the server sends `notifications/events/list_changed`.' + text: "The server's settings object has one member: `listChanged` (boolean, optional, default `false`) — whether the server sends `notifications/events/list_changed`." + - check: sep-9999-capability-empty-settings + text: 'An empty settings object declares event support with no list-change notifications.' + - check: sep-9999-capability-client-declaration + text: 'A client MAY advertise the same identifier with an empty settings object to indicate that it understands the extension.' + - check: sep-9999-capability-list-changed-gated + text: 'Like `notifications/tools/list_changed` and `notifications/resources/list_changed`, `notifications/events/list_changed` is sent only by a server that declared `listChanged: true` in its extension settings.' + - check: sep-9999-fallback-method-not-found + text: 'A server that does not offer the extension answers any `events/*` request with `-32601 MethodNotFound` (standard JSON-RPC).' # === Listing Available Events === - check: sep-9999-list-implemented From 5befc60a10b98de07d434bf7f1301caf47243469 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 00:33:59 +0000 Subject: [PATCH 12/27] feat(events): drive the fixture's diagnostic controls where they exist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six push and webhook rows describe what a server does when something goes wrong upstream, and a healthy server does none of it during a run. The scenarios watched a window, saw nothing, and reported untestable, which under src/scenarios/untestable.ts is red and stays red. A fixture MAY now expose the conditions as ordinary tools. This calls the two that are safe to fire mid-stream — a transient upstream failure and a retention gap — neither of which ends the subscription, so the rows that grade heartbeats and deliveries still see what they need. Termination is left alone deliberately: it is terminal for the source and these scenarios share one fixture process, so firing it here would poison whatever runs next. The controls are called over their own connection rather than the streaming one. That is the more faithful simulation, since an upstream failure does not arrive as a request from the subscriber watching for it, and the signals fan out to every live subscriber regardless. Discovery happens on the connection run() already holds, not around the stream. The first version probed later and cost a connect plus two tools/list round trips inside the observation window, which was enough to miss a termination arriving at 250ms and turned negative-push.test.ts red. A server without the controls now pays nothing for the question and opens no second connection. Servers without controls are unaffected and keep reporting untestable, with the reason naming the specific tool that would make the row gradeable rather than describing the gap in the abstract. Refs #504 --- src/scenarios/server/events/helpers.ts | 57 ++++++++++++++++++ src/scenarios/server/events/push.ts | 83 ++++++++++++++++++++++---- 2 files changed, 128 insertions(+), 12 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index f5e0c8b2..fe243d45 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -187,6 +187,63 @@ export async function declaredEventsCapability( return { declared: true, value: exts[EVENTS_EXTENSION_ID] }; } +/** + * Names of the optional diagnostic controls a fixture may expose so the harness + * can provoke conditions no protocol request can ask for. + * + * Several requirements describe what a server does when something goes wrong + * upstream, and a healthy server does none of those during a run. Without a way + * to trigger them the rows report untestable forever. A fixture that wants + * those rows graded registers these as ordinary tools; one that does not is + * unaffected and keeps reporting untestable with the prerequisite named. + * + * Each control takes `{ name: }`. mcpkit's + * examples/events/kitchen-sink registers them under --conformance-events; see + * that repo's examples/CONVENTIONS.md for the convention. + */ +export const EVENTS_CONTROL_YIELD_ERROR = 'events_conformance_yield_error'; +export const EVENTS_CONTROL_YIELD_GAP = 'events_conformance_yield_gap'; + +/** + * Whether the server exposes a given diagnostic control. + * + * Absence is the normal case and never an error: the caller falls back to + * reporting untestable with the missing prerequisite named, per + * src/scenarios/untestable.ts. + */ +export async function hasControl( + conn: Connection, + tool: string +): Promise { + try { + const res = await conn.request<{ tools?: { name?: string }[] }>( + 'tools/list' + ); + return (res.tools ?? []).some((t) => t?.name === tool); + } catch { + return false; + } +} + +/** + * Fire a diagnostic control, returning whether it ran. A server that lists the + * tool but rejects the call is treated as not having it, so a half-implemented + * control reports untestable rather than failing the requirement it was meant + * to exercise. + */ +export async function fireControl( + conn: Connection, + tool: string, + name: string +): Promise { + try { + await conn.request('tools/call', { name: tool, arguments: { name } }); + return true; + } catch { + return false; + } +} + /** * The `extensions` map from a capabilities object, or an empty map. * diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts index 460576c2..b2e2bbfb 100644 --- a/src/scenarios/server/events/push.ts +++ b/src/scenarios/server/events/push.ts @@ -15,13 +15,21 @@ * window would report a compliant slow-heartbeat server as broken, which is * worse than taking the time. * - * Several rows need a server doing something the harness cannot ask for: an - * upstream failure (`stream-error-is-recoverable`), a retention gap + * Several rows need a server doing something no protocol request can ask for: + * an upstream failure (`stream-error-is-recoverable`), a retention gap * (`stream-gap-resends-active`), a termination (`stream-terminated-*`), or a - * server-initiated close (`stream-final-result-*`). Those report untestable - * with the missing prerequisite named, per the untestable policy in - * src/scenarios/untestable.ts, rather than passing vacuously against a server - * that simply never did it. + * server-initiated close (`stream-final-result-*`). + * + * A fixture MAY expose diagnostic controls as ordinary tools, and this scenario + * calls the two that are safe to fire mid-stream: neither ends the + * subscription, so the rows below still see the heartbeats and deliveries they + * grade. Termination is not fired, because it is terminal for the source and + * these scenarios share one fixture process, so terminating here would poison + * whatever runs next. + * + * A server with no controls is unaffected and keeps reporting untestable with + * the missing prerequisite named, per src/scenarios/untestable.ts, rather than + * passing vacuously against a server that simply never did it. */ import { ClientScenario, ConformanceCheck } from '../../../types'; @@ -30,6 +38,10 @@ import { untestableCheck } from '../../untestable'; import { EVENTS_ACTIVE_NOTIFICATION, EVENTS_EXTENSION_ID, + EVENTS_CONTROL_YIELD_ERROR, + EVENTS_CONTROL_YIELD_GAP, + hasControl, + fireControl, extensionsOf, EVENTS_ERROR_NOTIFICATION, EVENTS_EVENT_NOTIFICATION, @@ -151,6 +163,16 @@ export class EventsPushScenario implements ClientScenario { ); } + // Probe for the diagnostic controls here, on the connection that is + // already open. A server without them must pay nothing for the question: + // doing this later, around the stream, cost two round trips inside the + // observation window and was enough to miss a termination arriving at + // 250ms. + const controls = { + error: await hasControl(conn, EVENTS_CONTROL_YIELD_ERROR), + gap: await hasControl(conn, EVENTS_CONTROL_YIELD_GAP) + }; + const args = minimalArguments(target); if (args === undefined) { return untestableAll( @@ -159,7 +181,7 @@ export class EventsPushScenario implements ClientScenario { ); } - return await this.streamChecks(ctx, name, args); + return await this.streamChecks(ctx, name, args, controls); } finally { await conn.close(); } @@ -168,7 +190,8 @@ export class EventsPushScenario implements ClientScenario { private async streamChecks( ctx: RunContext, name: string, - args: Record + args: Record, + controls: { error: boolean; gap: boolean } ): Promise { const checks: ConformanceCheck[] = []; const session = await openEventStream( @@ -221,12 +244,43 @@ export class EventsPushScenario implements ClientScenario { ); checks.push(this.activeCheck(active, session)); + // --- Provoke the conditions a healthy server never produces ---------- + // An upstream failure and a retention gap are both things the protocol + // gives a client no way to ask for, so against a server with no + // diagnostic controls these rows watch, see nothing, and report + // untestable. A fixture that registers the controls gets them graded. + // Fired inside the observation window so the frames land in this + // session's notification list alongside everything else. + // + // Neither control ends the subscription, which is what makes them safe + // to fire here: the stream stays open and the rows below still see the + // heartbeats and deliveries they grade. + // + // Fired over their own connection rather than the streaming one, which is + // also the more faithful simulation: an upstream failure does not arrive + // as a request from the subscriber watching for it. The signals fan out + // to every live subscriber of the event type, so they reach this session + // regardless. A server with no controls opens no connection here. + if (controls.error || controls.gap) { + const control = await ctx.connect(); + try { + if (controls.error) { + await fireControl(control, EVENTS_CONTROL_YIELD_ERROR, name); + } + if (controls.gap) { + await fireControl(control, EVENTS_CONTROL_YIELD_GAP, name); + } + } finally { + await control.close(); + } + } + // --- Watch the stream ------------------------------------------------ // One window serves every timing row: heartbeats, delivered events, and // whatever else the server chooses to put on the stream. await session.settle(WATCH_MS); - checks.push(...this.notificationChecks(session, name)); + checks.push(...this.notificationChecks(session, name, controls)); checks.push(...this.heartbeatChecks(session)); // --- Cancellation ------------------------------------------------------ @@ -320,7 +374,8 @@ export class EventsPushScenario implements ClientScenario { /** What rode the stream, and whether every frame was routable. */ private notificationChecks( session: StreamSession, - name: string + name: string, + controls: { error: boolean; gap: boolean } ): ConformanceCheck[] { const out: ConformanceCheck[] = []; const notifications = session.notifications; @@ -437,7 +492,9 @@ export class EventsPushScenario implements ClientScenario { 'sep-9999-stream-error-is-recoverable', 'sep-9999-stream-error-is-recoverable', '`notifications/events/error` reports a recoverable failure; the subscription remains active.', - 'No upstream failure occurred during the run, and the harness cannot provoke one. Needs a fixture whose upstream can be made to fail on demand.', + controls.error + ? `The \`${EVENTS_CONTROL_YIELD_ERROR}\` control was called for \`${name}\` but no \`${EVENTS_ERROR_NOTIFICATION}\` arrived, so the recovery path could not be observed.` + : `No upstream failure occurred during the run, and the harness cannot provoke one over the protocol. Needs a fixture exposing the \`${EVENTS_CONTROL_YIELD_ERROR}\` control.`, [EVENTS_SPEC_REF] ) : eventsCheck( @@ -489,7 +546,9 @@ export class EventsPushScenario implements ClientScenario { 'sep-9999-stream-gap-resends-active', 'sep-9999-stream-gap-resends-active', 'A gap is signalled by a fresh `notifications/events/active` with `truncated: true`, not an error.', - 'No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture that can expire its replay window on demand.', + controls.gap + ? `The \`${EVENTS_CONTROL_YIELD_GAP}\` control was called for \`${name}\` but no second \`active\` carrying \`truncated: true\` arrived.` + : `No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture exposing the \`${EVENTS_CONTROL_YIELD_GAP}\` control.`, [EVENTS_SPEC_REF], 'WARNING' ) From f9eb9a48566726d6e62a84d97135ed79935412cf Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 01:24:55 +0000 Subject: [PATCH 13/27] feat(events): grade the three rows that need a server-ended stream sep-9999-stream-terminated-ends-subscription and both sep-9999-stream-final-result-* could not be graded on the main session, and not for want of a control: the harness cancels that stream to test stream-cancel-stops-delivery, and on Streamable HTTP a client-side abort is terminal, so no final frame is ever sent. A server-ended stream is a different stream. terminateChecks opens its own, fires the terminate control, and grades all three from it. It refuses to terminate the event type the rest of the run depends on, because termination is one-shot for the source and the four events scenarios share one fixture process; a fixture offering a single push-capable type gets untestable with that reason rather than a poisoned suite. The main session keeps grading these opportunistically. A server that closes the stream itself has already answered them, and dedupe is first-wins, so the control path never runs for ids the observation window already produced. That ordering is what the negative fixture exercises, and getting it wrong turned negative-push.test.ts red twice: once by removing the opportunistic path entirely, once by flattening three rows to one severity when terminating is a MUST and the two final-result rows are SHOULDs. Also fixes a bug in the final-result shape check that mcpkit surfaced. It counted `resultType` as information the frame carries, but `resultType` is a base-protocol field on the common Result interface that servers MUST include, which sep-2640.yaml already records. Every conformant server failed that row. `_meta` was already excluded; `resultType` now is too. Against mcpkit: events-push 14/16 to 17/18, zero warnings, the remaining failure being a real missing subscriptionId. Refs #504 --- src/scenarios/server/events/helpers.ts | 6 + src/scenarios/server/events/push.ts | 208 ++++++++++++++++++++++--- 2 files changed, 190 insertions(+), 24 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index fe243d45..4b0a00ab 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -203,6 +203,12 @@ export async function declaredEventsCapability( */ export const EVENTS_CONTROL_YIELD_ERROR = 'events_conformance_yield_error'; export const EVENTS_CONTROL_YIELD_GAP = 'events_conformance_yield_gap'; +/** + * Ends every live subscription to an event type. Terminal for that type for + * the life of the fixture process, so a scenario firing it must pick a type + * nothing else in the run depends on. + */ +export const EVENTS_CONTROL_TERMINATE = 'events_conformance_terminate'; /** * Whether the server exposes a given diagnostic control. diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts index b2e2bbfb..cc6fcd07 100644 --- a/src/scenarios/server/events/push.ts +++ b/src/scenarios/server/events/push.ts @@ -40,6 +40,9 @@ import { EVENTS_EXTENSION_ID, EVENTS_CONTROL_YIELD_ERROR, EVENTS_CONTROL_YIELD_GAP, + EVENTS_CONTROL_TERMINATE, + deliveryModes, + type EventDescriptor, hasControl, fireControl, extensionsOf, @@ -69,6 +72,10 @@ const WATCH_MS = Number(process.env.EVENTS_PUSH_WATCH_MS ?? 35000); /** How long to wait for the subscription confirmation before grading it. */ const ACTIVE_MS = 3000; +// How long to wait for the terminated frame after firing the control. Generous +// against ACTIVE_MS because the control travels on its own connection and the +// fanout is asynchronous. +const TERMINATE_MS = 5000; /** Slack on the 30s heartbeat SHOULD, for scheduling and network jitter. */ const HEARTBEAT_TOLERANCE_MS = 2000; @@ -170,7 +177,8 @@ export class EventsPushScenario implements ClientScenario { // 250ms. const controls = { error: await hasControl(conn, EVENTS_CONTROL_YIELD_ERROR), - gap: await hasControl(conn, EVENTS_CONTROL_YIELD_GAP) + gap: await hasControl(conn, EVENTS_CONTROL_YIELD_GAP), + terminate: await hasControl(conn, EVENTS_CONTROL_TERMINATE) }; const args = minimalArguments(target); @@ -181,7 +189,13 @@ export class EventsPushScenario implements ClientScenario { ); } - return await this.streamChecks(ctx, name, args, controls); + return await this.streamChecks( + ctx, + name, + args, + controls, + listed.descriptors + ); } finally { await conn.close(); } @@ -191,7 +205,8 @@ export class EventsPushScenario implements ClientScenario { ctx: RunContext, name: string, args: Record, - controls: { error: boolean; gap: boolean } + controls: { error: boolean; gap: boolean; terminate: boolean }, + descriptors: EventDescriptor[] ): Promise { const checks: ConformanceCheck[] = []; const session = await openEventStream( @@ -310,7 +325,19 @@ export class EventsPushScenario implements ClientScenario { ) ); - checks.push(...this.finalResultChecks(session)); + // Opportunistic, and pushed first so dedupe keeps it: a server that + // closed this stream itself has already answered the final-result rows, + // and does not need the control path to provoke a second close. + checks.push(...this.finalResultChecks(session, false)); + + checks.push( + ...(await this.terminateChecks( + ctx, + descriptors, + name, + controls.terminate + )) + ); checks.push(...(await this.concurrencyChecks(ctx, name, args))); checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); return dedupe(checks); @@ -375,7 +402,7 @@ export class EventsPushScenario implements ClientScenario { private notificationChecks( session: StreamSession, name: string, - controls: { error: boolean; gap: boolean } + controls: { error: boolean; gap: boolean; terminate: boolean } ): ConformanceCheck[] { const out: ConformanceCheck[] = []; const notifications = session.notifications; @@ -510,25 +537,24 @@ export class EventsPushScenario implements ClientScenario { ) ); + // A server that terminates on its own during the main window is graded + // here and the control path below never runs for these ids, since dedupe + // is first-wins. Nothing is emitted when it does not: terminateChecks owns + // the untestable branch, because it is the one that knows whether a + // control existed and whether a spare event type was available. const terminated = notifications.filter( (n) => n.method === EVENTS_TERMINATED_NOTIFICATION ); - out.push( - terminated.length === 0 - ? untestableCheck( - 'sep-9999-stream-terminated-ends-subscription', - 'sep-9999-stream-terminated-ends-subscription', - 'Only `notifications/events/terminated` ends the subscription.', - 'The subscription was not terminated during the run. Needs a server that can revoke authorization or remove an event type mid-stream.', - [EVENTS_SPEC_REF] - ) - : eventsCheck( - 'sep-9999-stream-terminated-ends-subscription', - 'Only `notifications/events/terminated` ends the subscription.', - 'SUCCESS', - { details: { terminated: terminated.length } } - ) - ); + if (terminated.length > 0) { + out.push( + eventsCheck( + 'sep-9999-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + 'SUCCESS', + { details: { terminated: terminated.length } } + ) + ); + } const actives = notifications.filter( (n) => n.method === EVENTS_ACTIVE_NOTIFICATION @@ -679,21 +705,155 @@ export class EventsPushScenario implements ClientScenario { } /** The `StreamEventsResult`, which only a server-initiated close produces. */ - private finalResultChecks(session: StreamSession): ConformanceCheck[] { + /** + * Grade the three rows that need the *server* to end the stream: + * `stream-terminated-ends-subscription` and both `stream-final-result-*`. + * + * These cannot be graded on the main session. The harness cancels that one + * to test `stream-cancel-stops-delivery`, and on Streamable HTTP a + * client-side abort is terminal, so no final frame is ever sent. A + * server-ended stream is a different stream. + * + * Termination is one-shot for the event type and the events scenarios share + * a fixture process, so this deliberately refuses to terminate the type the + * rest of the run depends on. A fixture with only one push-capable type gets + * untestable rather than a poisoned suite. + */ + private async terminateChecks( + ctx: RunContext, + descriptors: EventDescriptor[], + usedName: string, + hasTerminate: boolean + ): Promise { + // Severity follows each row's own keyword, not the reason they share. + // Terminating is a MUST; the final-result shape and timing are SHOULDs, and + // flattening all three to FAILURE overstates two of them. + const untestableHere = (reason: string): ConformanceCheck[] => [ + ...untestableAll( + ['sep-9999-stream-terminated-ends-subscription'], + reason + ), + ...untestableAll( + [ + 'sep-9999-stream-final-result-shape', + 'sep-9999-stream-final-result-timing' + ], + reason, + 'WARNING' + ) + ]; + + if (!hasTerminate) { + return untestableHere( + `The subscription was not terminated during the run, and the harness cannot revoke one over the protocol. Needs a fixture exposing the \`${EVENTS_CONTROL_TERMINATE}\` control.` + ); + } + + const spare = descriptors + .filter((d) => deliveryModes(d).includes('push')) + .map((d) => descriptorName(d)) + .find((n): n is string => !!n && n !== usedName); + if (!spare) { + return untestableHere( + `Terminating an event type is one-shot for the life of the fixture, and \`${usedName}\` is the only push-capable type on offer, so terminating it would break every scenario after this one. Needs a second push-capable type the run does not otherwise depend on.` + ); + } + + const args = minimalArguments( + descriptors.find((d) => descriptorName(d) === spare)! + ); + if (args === undefined) { + return untestableHere( + `Event type \`${spare}\` declares required \`inputSchema\` properties the harness cannot satisfy, so no stream could be opened to terminate.` + ); + } + + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name: spare, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + try { + if (!session.contentType?.includes('text/event-stream')) { + return untestableHere( + `\`${EVENTS_STREAM_METHOD}\` did not open a stream for \`${spare}\`, so there was nothing to terminate.` + ); + } + await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + + const control = await ctx.connect(); + try { + await fireControl(control, EVENTS_CONTROL_TERMINATE, spare); + } finally { + await control.close(); + } + + const terminated = await session.waitFor( + (n) => n.method === EVENTS_TERMINATED_NOTIFICATION, + TERMINATE_MS + ); + await session.settle(500); + + const out: ConformanceCheck[] = []; + out.push( + terminated + ? eventsCheck( + 'sep-9999-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + session.open ? 'FAILURE' : 'SUCCESS', + { + errorMessage: session.open + ? `\`${EVENTS_TERMINATED_NOTIFICATION}\` arrived for \`${spare}\` but the stream stayed open.` + : undefined, + details: { name: spare } + } + ) + : eventsCheck( + 'sep-9999-stream-terminated-ends-subscription', + 'Only `notifications/events/terminated` ends the subscription.', + 'FAILURE', + { + errorMessage: `The \`${EVENTS_CONTROL_TERMINATE}\` control was called for \`${spare}\` but no \`${EVENTS_TERMINATED_NOTIFICATION}\` arrived within ${TERMINATE_MS}ms.` + } + ) + ); + out.push(...this.finalResultChecks(session)); + return out; + } finally { + await session.cancel(); + } + } + + private finalResultChecks( + session: StreamSession, + emitUntestable = true + ): ConformanceCheck[] { const result = session.finalResult?.result; if (result === undefined) { + // The main session calls this opportunistically: a server that closed + // the stream itself has answered these rows, and one that did not leaves + // them to terminateChecks, which knows why they could not be exercised. + if (!emitUntestable) return []; return untestableAll( [ 'sep-9999-stream-final-result-shape', 'sep-9999-stream-final-result-timing' ], - 'The harness cancelled the stream, and on Streamable HTTP a client-side abort is terminal, so no final frame is sent. Grading this needs a server that closes the stream itself.', + 'The server did not close the stream itself, so no final frame was sent.', 'WARNING' ); } const out: ConformanceCheck[] = []; + // `_meta` and `resultType` are base-protocol fields on the common `Result` + // interface, not information this extension's final frame carries. Servers + // MUST include `resultType`, so counting it as a payload field fails every + // conformant server; mcpkit is how that surfaced. const keys = isObject(result) - ? Object.keys(result).filter((k) => k !== '_meta') + ? Object.keys(result).filter((k) => k !== '_meta' && k !== 'resultType') : ['']; out.push( keys.length === 0 From 6be2ca1ac58d729d7880d08e2eacd860e8150add Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 01:49:14 +0000 Subject: [PATCH 14/27] feat(events): grade cross-tenant isolation via the fixture's tenant controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The rule is that two tenants subscribing to the same `(name, arguments)` with the same callback get distinct subscriptions, because the principal is part of the key. A run authenticates as one principal for its lifetime, so the harness could only ever supply one side of that comparison and the row reported untestable. A fixture MAY now register on another principal's behalf and answer whether a given principal's subscription still exists. With both, the check does the comparison the rule is about: same event, same callback, principal the only thing that differs. Two things are graded, and the second is the one that bites. Distinct ids show the principal reached the key at all. A surviving subscription after the other tenant unsubscribes shows the two are genuinely independent, which is what a tenant would notice if it were not true — one tenant able to cancel another's subscription. askControl joins fireControl in helpers for the controls that answer with text rather than just succeeding. Servers without the controls are unaffected and keep reporting untestable, now naming the two tools that would make the row gradeable. Against mcpkit: events-webhook 17/23 to 18/23. Refs #504 --- src/scenarios/server/events/helpers.ts | 31 +++++++ src/scenarios/server/events/webhook.ts | 118 +++++++++++++++++++++++-- 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 4b0a00ab..27853268 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -209,6 +209,37 @@ export const EVENTS_CONTROL_YIELD_GAP = 'events_conformance_yield_gap'; * nothing else in the run depends on. */ export const EVENTS_CONTROL_TERMINATE = 'events_conformance_terminate'; +/** + * Registers a subscription on another principal's behalf, returning its derived + * id. A run authenticates as one principal for its lifetime, so this is the + * only way to construct the two-tenant case the key-composition rule is about. + */ +export const EVENTS_CONTROL_SUBSCRIBE_AS = 'events_conformance_subscribe_as'; +/** Reports whether a given principal's subscription is still registered. */ +export const EVENTS_CONTROL_SUBSCRIPTION_EXISTS = + 'events_conformance_subscription_exists'; + +/** + * Fire a control that answers with text, returning the text or undefined. + * Distinct from fireControl, which only cares that the call succeeded. + */ +export async function askControl( + conn: Connection, + tool: string, + args: Record +): Promise { + try { + const res = await conn.request<{ + content?: { type?: string; text?: string }[]; + isError?: boolean; + }>('tools/call', { name: tool, arguments: args }); + if (res.isError) return undefined; + const text = (res.content ?? []).find((c) => c?.type === 'text')?.text; + return typeof text === 'string' ? text : undefined; + } catch { + return undefined; + } +} /** * Whether the server exposes a given diagnostic control. diff --git a/src/scenarios/server/events/webhook.ts b/src/scenarios/server/events/webhook.ts index 933ce7a7..9fb9409d 100644 --- a/src/scenarios/server/events/webhook.ts +++ b/src/scenarios/server/events/webhook.ts @@ -33,6 +33,10 @@ import { JsonRpcError } from '../../../connection'; import { untestableCheck } from '../../untestable'; import { EVENTS_EXTENSION_ID, + EVENTS_CONTROL_SUBSCRIBE_AS, + EVENTS_CONTROL_SUBSCRIPTION_EXISTS, + hasControl, + askControl, extensionsOf, EVENTS_NOT_FOUND, EVENTS_SPEC_REF, @@ -323,6 +327,7 @@ export class EventsWebhookScenario implements ClientScenario { ); checks.push( ...(await this.identityChecks( + conn, subscribe, release, first.result, @@ -793,7 +798,110 @@ export class EventsWebhookScenario implements ClientScenario { } /** The compound key, the derived id, and what a refresh does. */ + /** + * Grade sep-9999-subscribe-cross-tenant-isolation, which asserts that two + * tenants subscribing to the same `(name, arguments)` with the same callback + * get distinct subscriptions, because the principal is part of the key. + * + * A run authenticates as one principal for its lifetime, so the harness can + * only ever supply one side of that comparison. The other side comes from a + * fixture control that registers on another principal's behalf, using the + * same identity function the subscribe handler uses. + * + * Two things are checked, and the second is the one that bites. Distinct ids + * show the principal reached the key at all. A surviving subscription after + * the other tenant unsubscribes shows the two are genuinely independent, + * which is what a tenant would actually notice. + */ + private async crossTenantChecks( + conn: Connection, + name: string, + subscribe: ( + params: Record + ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }> + ): Promise { + const id = 'sep-9999-subscribe-cross-tenant-isolation'; + const description = + 'Because the key includes `principal` and `delivery.url`, two distinct tenants subscribing to the same `(name, arguments)` get distinct subscriptions.'; + + const canAct = await hasControl(conn, EVENTS_CONTROL_SUBSCRIBE_AS); + const canAsk = await hasControl(conn, EVENTS_CONTROL_SUBSCRIPTION_EXISTS); + if (!canAct || !canAsk) { + return [ + untestableCheck( + id, + id, + description, + `A run holds one principal, so the two-tenant case cannot be constructed. Needs a fixture exposing the \`${EVENTS_CONTROL_SUBSCRIBE_AS}\` and \`${EVENTS_CONTROL_SUBSCRIPTION_EXISTS}\` controls. The \`delivery.url\` half of the same rule is graded by sep-9999-subscribe-key-composition.`, + [EVENTS_SPEC_REF] + ) + ]; + } + + // Same event, same callback. The principal is the only thing that differs, + // which is what makes a shared id mean what it means. + const url = `${CALLBACK_BASE}/cross-tenant`; + const other = { principal: 'conformance-tenant-b', name, url }; + + const otherId = await askControl(conn, EVENTS_CONTROL_SUBSCRIBE_AS, other); + if (!otherId) { + return [ + untestableCheck( + id, + id, + description, + `The \`${EVENTS_CONTROL_SUBSCRIBE_AS}\` control did not return a subscription id, so the second tenant could not be established.`, + [EVENTS_SPEC_REF] + ) + ]; + } + + const mine = await subscribe({ + name, + arguments: {}, + delivery: { mode: 'webhook', url, secret: freshSecret() } + }); + if ('error' in mine) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `Subscribing as this run's own principal to the same event and callback failed with ${mine.error.code} ${mine.error.message}, so the two subscriptions could not be compared.` + }) + ]; + } + + const myId = mine.result.id; + if (myId === otherId) { + return [ + eventsCheck(id, description, 'FAILURE', { + errorMessage: `Two principals subscribing to \`${name}\` with the same callback received the same subscription id (${myId}), so the principal is not part of the key. One tenant can address, refresh or cancel another's subscription.`, + details: { id: myId } + }) + ]; + } + + await conn + .request(EVENTS_UNSUBSCRIBE_METHOD, { id: myId }) + .catch(() => undefined); + const survived = await askControl( + conn, + EVENTS_CONTROL_SUBSCRIPTION_EXISTS, + other + ); + + return [ + survived === 'true' + ? eventsCheck(id, description, 'SUCCESS', { + details: { mine: myId, other: otherId } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `The two tenants received distinct ids, but unsubscribing \`${myId}\` also removed the other tenant's subscription, so they are not independent.`, + details: { mine: myId, other: otherId } + }) + ]; + } + private async identityChecks( + conn: Connection, subscribe: ( p: Record ) => Promise<{ result: SubscribeResult } | { error: JsonRpcError }>, @@ -993,15 +1101,7 @@ export class EventsWebhookScenario implements ClientScenario { ) ); - out.push( - untestableCheck( - 'sep-9999-subscribe-cross-tenant-isolation', - 'sep-9999-subscribe-cross-tenant-isolation', - 'Because the key includes `principal` and `delivery.url`, two distinct tenants subscribing to the same `(name, arguments)` get distinct subscriptions.', - 'A run holds one principal, so the two-tenant case cannot be constructed. The `delivery.url` half of the same rule is graded by sep-9999-subscribe-key-composition.', - [EVENTS_SPEC_REF] - ) - ); + out.push(...(await this.crossTenantChecks(conn, name, subscribe))); out.push( untestableCheck( From 80d652864e9c4c6c9173f1c99211414e75611297 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 06:29:45 +0000 Subject: [PATCH 15/27] fix(events): probes answer the verification challenge, and grade a failed one Once a server verifies inside events/subscribe, which is how the document reads ("returned synchronously from events/subscribe"), three things in the webhook-delivery scenario stopped measuring what they claim to. Every probe callback answered every POST with its failure status, the challenge included, so a verifying server could not subscribe the 410, 413, retry or redirect probes and all four rows went untestable. The receiver now answers the challenge on every path before the path's behaviour applies; wrong-challenge is the one path that still fails it. The 410/413 and retry graders also waited for the first POST on their path, which is now the challenge, answered 200 and never retried, so they wait for the first non-verification POST instead. The redirect grader waits for a POST it actually answered with 302. verification-failure-error was untestable on the assumption that the handshake is asynchronous. It now subscribes a wrong-challenge callback: -32015 with reason challenge_failed passes; any other error, or an accepted subscription that is then delivered to, fails; an accepted subscription with no delivery is a WARNING, since the endpoint is safe but the subscriber was never told. delivery-retry-regenerates-signature failed on any two retries inside one second, because webhook-timestamp is whole seconds and a regenerated stamp is indistinguishable from a reused one there. It flapped against mcpkit, whose first retry is 500ms out. Only a stamp repeated across attempts a second or more apart now fails; retries all inside one second are a WARNING. The fixture gains synchronousVerification (and ignoreFailedEcho for the negative case) and retryGapMs. Against mcpkit kitchen-sink --serve the scenario goes from 12 to 18 SUCCESS; with verification disabled in the library the verification rows go red again. --- .../server/events/negative-delivery.test.ts | 91 ++++++++++ .../server/events/negative-fixture.ts | 70 +++++++- src/scenarios/server/events/receiver.ts | 20 ++- .../server/events/webhook-delivery.ts | 170 ++++++++++++++---- 4 files changed, 310 insertions(+), 41 deletions(-) diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index 6c9be756..a28b4dbf 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -208,6 +208,84 @@ describe.concurrent('the verification handshake', () => { TIMEOUT ); + // A server that verifies inside events/subscribe, the way the document's + // "returned synchronously from events/subscribe" reads, refuses a callback + // that echoes the wrong nonce with -32015 challenge_failed. + test( + 'refusing a wrong echo with -32015 challenge_failed passes the failure row', + async () => { + const checks = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + const check = checks.get('sep-9999-verification-failure-error'); + expect(check?.status).toBe('SUCCESS'); + }, + TIMEOUT + ); + + test( + 'accepting a subscription whose echo was wrong fails the failure row', + async () => { + const checks = await deliveryChecks( + delivering({ synchronousVerification: true, ignoreFailedEcho: true }) + ); + const check = checks.get('sep-9999-verification-failure-error'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32015'); + }, + TIMEOUT + ); + + // The default fixture challenges after subscribe returns and ignores the + // answer, so it delivers to an endpoint that never consented. + test( + 'delivering after a wrong echo fails the failure row', + async () => { + const checks = await deliveryChecks(delivering()); + const check = checks.get('sep-9999-verification-failure-error'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('delivered'); + }, + TIMEOUT + ); + + test( + 'with no challenge at all, the failure row stays untestable', + async () => { + const checks = await deliveryChecks(delivering({ verify: false })); + expect( + checks.get('sep-9999-verification-failure-error')?.details?.untestable + ).toBe(true); + }, + TIMEOUT + ); + + // The probe callbacks used to answer every POST with their failure status, + // the challenge included, so a server that verifies synchronously could not + // subscribe them and every probe row went untestable. + test( + 'a synchronous verifier still has its probe rows graded', + async () => { + const checks = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + for (const id of [ + 'sep-9999-delivery-410-non-retryable', + 'sep-9999-delivery-413-non-retryable', + 'sep-9999-delivery-retries-bounded', + 'sep-9999-ssrf-no-redirects' + ]) { + const check = checks.get(id); + expect( + check?.details?.untestable, + `${id}: ${check?.errorMessage}` + ).not.toBe(true); + expect(check?.status, `${id}: ${check?.errorMessage}`).toBe('SUCCESS'); + } + }, + TIMEOUT + ); + test( 'delivering an event before the challenge fails', async () => { @@ -330,6 +408,19 @@ describe.concurrent('retries and redirects', () => { TIMEOUT ); + // webhook-timestamp is in whole seconds, so two attempts inside one second + // carry the same stamp whether or not the server regenerated it. mcpkit's + // first retry is 500ms out, which made this row flap on it. + test( + 'sub-second retries sharing a stamp do not fail the freshness row', + async () => { + const checks = await deliveryChecks(delivering({ retryGapMs: 300 })); + const check = checks.get('sep-9999-delivery-retry-regenerates-signature'); + expect(check?.status, check?.errorMessage).not.toBe('FAILURE'); + }, + TIMEOUT + ); + test( 'retrying after 410 and 413 fails both non-retryable rows', async () => { diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index 6b995cd7..cbf55a05 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -189,6 +189,15 @@ export interface SubscribeBehaviour { export interface DeliveryBehaviour { /** Send the verification envelope before any event. */ verify?: boolean; + /** + * Run the handshake inside `events/subscribe`, before answering, and refuse + * with -32015 `challenge_failed` when the callback does not echo the nonce + * in a 2xx body. The default challenges after the response and ignores the + * answer. + */ + synchronousVerification?: boolean; + /** With `synchronousVerification`, accept the subscription anyway. */ + ignoreFailedEcho?: boolean; /** Deliver an event at all. */ sendEvent?: boolean; /** Deliver the event before the verification envelope. */ @@ -211,6 +220,8 @@ export interface DeliveryBehaviour { followRedirects?: boolean; /** Total attempts for a delivery the callback rejects with 5xx. */ attempts?: number; + /** Milliseconds between retry attempts; defaults to just over a second. */ + retryGapMs?: number; /** Reuse the first attempt's timestamp and signature on every retry. */ staleRetrySignature?: boolean; /** Retry after 410 and 413, which the document defines as non-retryable. */ @@ -526,6 +537,16 @@ export async function startEventsFixture( refreshBefore = new Date(Date.now() + granted).toISOString(); } + if (opts.delivery?.synchronousVerification && typeof url === 'string') { + const echoed = await challengeCallback(url, delivery.secret, id); + if (!echoed && !opts.delivery.ignoreFailedEcho) { + fail(-32015, 'endpoint verification failed', { + reason: 'challenge_failed' + }); + return; + } + } + send({ ...(behaviour.omitId ? {} : { id }), refreshBefore, @@ -542,7 +563,9 @@ export async function startEventsFixture( delivery.secret, id, String(name), - opts.delivery + opts.delivery.synchronousVerification + ? { ...opts.delivery, verify: false } + : opts.delivery ).finally(() => inFlight.delete(run)); inFlight.add(run); } @@ -746,6 +769,47 @@ function isRecord(value: unknown): value is Record { * two attempts inside one second would share a stamp the fixture did freshen. */ const RETRY_GAP_MS = 1100; +/** + * The handshake as a server that verifies inside `events/subscribe` runs it: + * one signed `verification` POST, and true only for a 2xx whose body echoes the + * nonce. No retries and no redirects, the same as a delivery. + */ +async function challengeCallback( + url: string, + secret: unknown, + subscriptionId: string +): Promise { + const key = + typeof secret === 'string' && secret.startsWith('whsec_') + ? Buffer.from(secret.slice('whsec_'.length), 'base64') + : Buffer.from(String(secret ?? '')); + const challenge = `chal_${Math.random().toString(36).slice(2, 14)}`; + const body = JSON.stringify({ type: 'verification', challenge }); + const webhookId = `msg_verification_${Math.random().toString(36).slice(2, 10)}`; + const stamp = String(Math.floor(Date.now() / 1000)); + const signature = `v1,${createHmac('sha256', key).update(`${webhookId}.${stamp}.${body}`).digest('base64')}`; + try { + const res = await fetch(url, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'webhook-id': webhookId, + 'webhook-timestamp': stamp, + 'webhook-signature': signature, + 'x-mcp-subscription-id': subscriptionId + }, + body, + redirect: 'manual' + }); + const text = await res.text(); + if (res.status < 200 || res.status >= 300) return false; + const reply: unknown = JSON.parse(text); + return isRecord(reply) && reply.challenge === challenge; + } catch { + return false; + } +} + /** * POST the verification challenge and then the event, the way the document says * to, with whatever this behaviour breaks. @@ -830,7 +894,9 @@ async function deliverToCallback( } if (opts.retryable === false) return; if (attempt === attempts) return; - await new Promise((resolve) => setTimeout(resolve, RETRY_GAP_MS)); + await new Promise((resolve) => + setTimeout(resolve, behaviour.retryGapMs ?? RETRY_GAP_MS) + ); } }; diff --git a/src/scenarios/server/events/receiver.ts b/src/scenarios/server/events/receiver.ts index 7c966c07..6c851eaa 100644 --- a/src/scenarios/server/events/receiver.ts +++ b/src/scenarios/server/events/receiver.ts @@ -11,6 +11,13 @@ * Per-path behaviour lets one receiver serve every probe: a path that echoes * the challenge and accepts, one that redirects, one that refuses permanently, * and one that fails a few times before accepting. + * + * Every path answers the verification challenge first, whatever its behaviour, + * because a probe exists to misbehave on deliveries. A 410 probe that also + * refused the challenge would never get subscribed by a server that verifies + * inside events/subscribe, and its row would go untestable for a reason that + * has nothing to do with 410. `wrong-challenge` is the one path that fails the + * handshake, since that is its whole job. */ import http from 'node:http'; @@ -97,6 +104,14 @@ export async function startReceiver(host = '127.0.0.1'): Promise { res.end(body ?? '{}'); }; + const challenge = json?.challenge; + const isChallenge = + json?.type === 'verification' && typeof challenge === 'string'; + if (isChallenge && behaviour.kind !== 'wrong-challenge') { + respond(200, JSON.stringify({ challenge })); + return; + } + switch (behaviour.kind) { case 'redirect': res.writeHead(302, { location: behaviour.to }); @@ -133,8 +148,9 @@ export async function startReceiver(host = '127.0.0.1'): Promise { break; } - // The verification handshake: prove intent by echoing the nonce. - const challenge = json?.challenge; + // A challenge without the `type` discriminator: still echo it, so a + // server that got the envelope shape slightly wrong is graded on the + // row about the shape rather than locked out of every other row. if (typeof challenge === 'string') { respond(200, JSON.stringify({ challenge })); return; diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index d74faa11..80cf6845 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -390,6 +390,11 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } checks.push(...this.verificationChecks(all, subscriptionId)); + if (all.some(isVerificationEnvelope)) { + checks.push( + await this.verificationFailureCheck(receiver, subscribe, release) + ); + } checks.push(...this.transportChecks(all, subscriptionId)); checks.push(...this.signatureChecks(all, secret.bytes)); checks.push(...this.envelopeChecks(all)); @@ -470,10 +475,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { subscriptionId: unknown ): ConformanceCheck[] { const out: ConformanceCheck[] = []; - const verification = all.find( - (d) => - d.json?.type === 'verification' || typeof d.json?.challenge === 'string' - ); + const verification = all.find(isVerificationEnvelope); const events = all.filter((d) => typeof d.json?.eventId === 'string'); if (!verification) { @@ -535,16 +537,6 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } ) ); - out.push( - untestableCheck( - 'sep-9999-verification-failure-error', - 'sep-9999-verification-failure-error', - 'A reachable endpoint that fails to echo yields `-32015 CallbackEndpointError` with `data.reason: "challenge_failed"`.', - `Observing it needs the failure surfaced on a later subscribe, since the handshake is asynchronous. The harness echoes correctly here to reach the delivery rows; a dedicated probe against a non-echoing path belongs in a follow-up, and ${EVENTS_CALLBACK_ENDPOINT_ERROR} is the code to expect.`, - [EVENTS_SPEC_REF], - 'WARNING' - ) - ); } out.push( @@ -932,6 +924,65 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { return out; } + /** + * A callback that answers the challenge with the wrong nonce. The document + * has the failure come back from events/subscribe itself as -32015 with + * `data.reason: "challenge_failed"`. A server that accepts instead and then + * delivers there has sent events to an endpoint that never consented; one + * that accepts and withholds delivery verified asynchronously, which keeps + * the endpoint safe but reports nothing to the subscriber. + */ + private async verificationFailureCheck( + receiver: Receiver, + subscribe: ( + url: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (url: string) => Promise + ): Promise { + const id = 'sep-9999-verification-failure-error'; + const description = + 'A reachable endpoint that fails to echo yields `-32015 CallbackEndpointError` with `data.reason: "challenge_failed"`.'; + const path = `/wrong-challenge-${Date.now()}`; + receiver.behave(path, { kind: 'wrong-challenge' }); + const url = this.callbackFor(receiver, path); + const probe = await subscribe(url); + + if ('error' in probe) { + const { code, message, data } = probe.error; + const reason = isObject(data) ? data.reason : undefined; + if ( + code === EVENTS_CALLBACK_ENDPOINT_ERROR && + reason === 'challenge_failed' + ) { + return eventsCheck(id, description, 'SUCCESS', { + details: { code, reason } + }); + } + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Subscribing a callback that echoed the wrong nonce answered ${code} ${message} with data.reason ${describeValue(reason)}, expected ${EVENTS_CALLBACK_ENDPOINT_ERROR} with "challenge_failed".`, + details: { code, data } + }); + } + + try { + const delivered = await receiver.waitFor( + path, + (d) => typeof d.json?.eventId === 'string', + DELIVERY_WAIT_MS + ); + if (delivered) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce, and an event was then delivered to it. The document has this refused with ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" from events/subscribe, and no delivery to an endpoint that did not consent.` + }); + } + return eventsCheck(id, description, 'WARNING', { + errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce. Nothing was delivered to it, so the endpoint is safe, but the subscriber was never told: the document returns ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" synchronously from events/subscribe.` + }); + } finally { + await release(url); + } + } + /** A callback that redirects: the server must not follow it. */ private async redirectChecks( receiver: Receiver, @@ -965,7 +1016,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { try { const redirected = await receiver.waitFor( from, - () => true, + (d) => d.respondedStatus === 302, DELIVERY_WAIT_MS ); if (!redirected) { @@ -1029,7 +1080,11 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ); } else { try { - await receiver.waitFor(flaky, () => true, DELIVERY_WAIT_MS); + await receiver.waitFor( + flaky, + (d) => !isVerificationEnvelope(d), + DELIVERY_WAIT_MS + ); // Let the retries play out. await new Promise((resolve) => setTimeout(resolve, SETTLE_MS)); const attempts = receiver.on(flaky); @@ -1055,31 +1110,64 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { const stamps = retried.map( (a) => a.headers['webhook-timestamp'] ?? '' ); - const sigs = retried.map((a) => a.headers['webhook-signature'] ?? ''); // Freshness only: whether the signature verifies at all belongs to // sep-9999-delivery-signature-formula, and folding the two together // reports a server with a wrong formula as reusing timestamps it // plainly did not reuse. - const freshened = - new Set(stamps).size === stamps.length && - new Set(sigs).size === sigs.length; - out.push( - freshened - ? eventsCheck( - 'sep-9999-delivery-retry-regenerates-signature', - "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window.", - 'SUCCESS', - { details: { attempts: retried.length, stamps } } - ) - : eventsCheck( - 'sep-9999-delivery-retry-regenerates-signature', - "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window.", - 'FAILURE', - { - errorMessage: `Retries of the same \`webhook-id\` reused a timestamp or signature (timestamps ${stamps.join(', ')}), so a receiver enforcing the 5-minute freshness window would reject them.` - } - ) + // + // webhook-timestamp is whole seconds, so two attempts inside one + // second share a stamp whether or not it was regenerated. Only a + // stamp that stays put across attempts a second or more apart is + // provably reused. + const pairs = retried + .slice(1) + .map((a, i) => ({ prev: retried[i], next: a })); + const spaced = pairs.filter( + ({ prev, next }) => next.atMs - prev.atMs >= 1000 + ); + const reused = spaced.filter( + ({ prev, next }) => + (prev.headers['webhook-timestamp'] ?? '') === + (next.headers['webhook-timestamp'] ?? '') ); + const description = + "Each retry attempt MUST regenerate the timestamp and signature so retries are not rejected by the receiver's freshness window."; + if (reused.length > 0) { + out.push( + eventsCheck( + 'sep-9999-delivery-retry-regenerates-signature', + description, + 'FAILURE', + { + errorMessage: `Retries of the same \`webhook-id\` reused a timestamp or signature (timestamps ${stamps.join(', ')}, the repeat at least a second apart), so a receiver enforcing the 5-minute freshness window would reject them.` + } + ) + ); + } else if ( + spaced.length > 0 || + new Set(stamps).size === stamps.length + ) { + out.push( + eventsCheck( + 'sep-9999-delivery-retry-regenerates-signature', + description, + 'SUCCESS', + { details: { attempts: retried.length, stamps } } + ) + ); + } else { + out.push( + eventsCheck( + 'sep-9999-delivery-retry-regenerates-signature', + description, + 'WARNING', + { + errorMessage: `Every retry arrived within a second of the one before (timestamps ${stamps.join(', ')}), and webhook-timestamp is in whole seconds, so a regenerated stamp and a reused one look the same.`, + details: { attempts: retried.length, stamps } + } + ) + ); + } out.push( retried.length <= 6 ? eventsCheck( @@ -1135,9 +1223,11 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { continue; } try { + // The challenge on this path is answered 200 and never retried, so + // grading it would pass any server; the probe is about the event. const first = await receiver.waitFor( path, - () => true, + (d) => !isVerificationEnvelope(d), DELIVERY_WAIT_MS ); if (!first) { @@ -1176,6 +1266,12 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } } +function isVerificationEnvelope(d: ReceivedDelivery): boolean { + return ( + d.json?.type === 'verification' || typeof d.json?.challenge === 'string' + ); +} + /** Keep the first check emitted per id, so a fallback path cannot double-report. */ function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { const seen = new Set(); From 77c10e077e73a3d73be24dfc5e1e4bce6dc0a7ca Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 14:02:53 +0000 Subject: [PATCH 16/27] feat(events): grade the TTL durability rows through a restart control ttl-long-grant-retained, ttl-no-expiry-persisted and ttl-no-expiry-gc-terminated were untestable because they need a restart the harness cannot ask for over the wire. A fixture exposing three controls now makes them gradeable: - events_conformance_restart rebuilds the server over the same subscription store and answers the generation it is moving to - events_conformance_generation answers the current one - events_conformance_subscription_state answers active, suspended or absent for a derived id, which the exists control cannot, since it hides suspended subscriptions The generation pair is what makes the restart detectable on both kinds of server: a stateful one ends the old session, so the scenario reconnects; a stateless one keeps answering on the same connection, first from the old build and then the new. The scenario subscribes once with ttlMs:null and once with a one-day grant, restarts, and reads both back. A no-expiry grant that is not honoured leaves its two rows untestable at WARNING rather than failing, since granting one is optional. The GC row watches the no-expiry subscription, whose placeholder callback fails every delivery, for EVENTS_GC_WAIT_MS: dropped passes, still there is a WARNING because dropping is a MAY, and already lost across the restart is untestable. The terminated envelope goes to the placeholder origin and cannot be observed, which the check says. The durability checks run last, since a restart ends the scenario's connection. Wait limits are read at call time so the negative controls can shorten them. Against mcpkit kitchen-sink --conformance-events the scenario goes from 22/23 to 25/26; a build whose restart loses the store fails the two retention rows. --- src/scenarios/server/events/helpers.ts | 21 ++ .../server/events/negative-fixture.ts | 89 ++++++- .../server/events/negative-webhook.test.ts | 74 +++++- src/scenarios/server/events/webhook.ts | 240 +++++++++++++++++- 4 files changed, 420 insertions(+), 4 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 27853268..f7b1999e 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -218,6 +218,27 @@ export const EVENTS_CONTROL_SUBSCRIBE_AS = 'events_conformance_subscribe_as'; /** Reports whether a given principal's subscription is still registered. */ export const EVENTS_CONTROL_SUBSCRIPTION_EXISTS = 'events_conformance_subscription_exists'; +/** + * Rebuilds the server and its subscription registry over the same store, as a + * process restart would. Every session ends, so a scenario firing it must + * reconnect and must fire it last. + */ +export const EVENTS_CONTROL_RESTART = 'events_conformance_restart'; +/** + * Answers the fixture's current restart generation. The restart control + * answers the generation it is moving to, so a scenario can tell the restart + * happened whether the server keeps sessions (the old one dies) or is + * stateless (the same connection starts reaching the new build). + */ +export const EVENTS_CONTROL_GENERATION = 'events_conformance_generation'; +/** + * Takes `{ id }`, a derived subscription id, and answers `active`, + * `suspended` or `absent`. Unlike the exists control it sees a subscription + * the server has suspended after delivery failures, which is what separates + * "paused" from "dropped". + */ +export const EVENTS_CONTROL_SUBSCRIPTION_STATE = + 'events_conformance_subscription_state'; /** * Fire a control that answers with text, returning the text or undefined. diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index cbf55a05..38fad74b 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -236,7 +236,25 @@ export interface DeliveryBehaviour { terminatedEnvelope?: boolean | { error?: unknown }; } +/** + * The restart, generation and subscription-state controls, for the TTL + * durability rows. Defaults are conformant: both grants survive a restart and + * nothing is garbage-collected. + */ +export interface DurabilityBehaviour { + /** Drop a no-expiry subscription this long after it was created. */ + gcAfterMs?: number; + /** Lose no-expiry subscriptions across a restart. */ + dropNoExpiryOnRestart?: boolean; + /** Lose finite subscriptions across a restart. */ + dropFiniteOnRestart?: boolean; + /** Answer the restart control without restarting. */ + restartNoop?: boolean; +} + export interface EventsFixtureOptions { + /** Expose the durability controls as tools. */ + durability?: DurabilityBehaviour; /** * Raw value to declare at `capabilities.extensions["io.modelcontextprotocol/events"]`; * omit for no declaration. @@ -306,7 +324,11 @@ export async function startEventsFixture( const streams: Array> = []; const subscribes: Array> = []; /** Live subscriptions, keyed the way the document keys them. */ - const subscriptions = new Map(); + const subscriptions = new Map< + string, + { id: string; noExpiry?: boolean; at?: number } + >(); + let generation = 1; /** Deliveries still in flight, so close() can settle rather than abandon. */ const inFlight = new Set>(); let mintedIds = 0; @@ -349,6 +371,66 @@ export async function startEventsFixture( ); }; + if (opts.durability && method === 'tools/list') { + const obj = { type: 'object', properties: {} }; + send({ + tools: [ + { name: 'events_conformance_restart', inputSchema: obj }, + { name: 'events_conformance_generation', inputSchema: obj }, + { name: 'events_conformance_subscription_state', inputSchema: obj } + ] + }); + return; + } + if (opts.durability && method === 'tools/call') { + const d = opts.durability; + const text = (t: string) => + send({ content: [{ type: 'text', text: t }] }); + const tool = params.name; + const toolArgs = (params.arguments ?? {}) as Record; + if (tool === 'events_conformance_restart') { + if (d.restartNoop) { + text(String(generation + 1)); + return; + } + generation++; + for (const [key, sub] of [...subscriptions.entries()]) { + if (sub.noExpiry ? d.dropNoExpiryOnRestart : d.dropFiniteOnRestart) { + subscriptions.delete(key); + } + } + text(String(generation)); + return; + } + if (tool === 'events_conformance_generation') { + text(String(generation)); + return; + } + if (tool === 'events_conformance_subscription_state') { + const entry = [...subscriptions.entries()].find( + ([, sub]) => sub.id === toolArgs.id + ); + if (!entry) { + text('absent'); + return; + } + const [key, sub] = entry; + if ( + d.gcAfterMs !== undefined && + sub.noExpiry && + Date.now() - (sub.at ?? 0) > d.gcAfterMs + ) { + subscriptions.delete(key); + text('absent'); + return; + } + text('active'); + return; + } + fail(-32602, `unknown tool ${String(tool)}`); + return; + } + if (method === 'server/discover') { send({ supportedVersions: [DRAFT_PROTOCOL_VERSION], @@ -536,6 +618,11 @@ export async function startEventsFixture( : Math.min(Number(ttl), cap); refreshBefore = new Date(Date.now() + granted).toISOString(); } + const held = subscriptions.get(effectiveKey); + if (held) { + held.noExpiry = refreshBefore === null; + held.at = held.at ?? Date.now(); + } if (opts.delivery?.synchronousVerification && typeof url === 'string') { const echoed = await challengeCallback(url, delivery.secret, id); diff --git a/src/scenarios/server/events/negative-webhook.test.ts b/src/scenarios/server/events/negative-webhook.test.ts index 7bd8d366..bf9e23d8 100644 --- a/src/scenarios/server/events/negative-webhook.test.ts +++ b/src/scenarios/server/events/negative-webhook.test.ts @@ -1,4 +1,4 @@ -import { describe, test, expect } from 'vitest'; +import { describe, test, expect, beforeAll, afterAll, vi } from 'vitest'; import { testContext } from '../../../connection/testing'; import { DRAFT_PROTOCOL_VERSION } from '../../../types'; import { takeWireViolations } from '../../../validation/wire-schema'; @@ -27,6 +27,16 @@ import { const ALL_ROWS = 27; +// The durability rows poll for a restart and for a GC drop; the defaults are +// sized for a real server and would make this file crawl. +beforeAll(() => { + vi.stubEnv('EVENTS_RESTART_WAIT_MS', '1500'); + vi.stubEnv('EVENTS_GC_WAIT_MS', '2000'); +}); +afterAll(() => { + vi.unstubAllEnvs(); +}); + /** Rows every conformant run passes, which is the whole surface bar the seven * that need a second principal or a restart. */ const GRADEABLE = [ @@ -411,3 +421,65 @@ describe('the unsupported delivery mode', () => { expect(check?.errorMessage).toContain('no type to probe'); }); }); + +/** + * The durability rows, through a fixture exposing the restart, generation + * and subscription-state controls. The fixture is stateless, so a restart + * leaves the connection working; the generation control is what tells the + * scenario the restart happened. + */ +describe.concurrent('durability across a restart', () => { + const durable = ( + durability: NonNullable + ): EventsFixtureOptions => ({ ...webhookFixture(), durability }); + + test('a store that keeps both grants and GCs the failing one passes all three', async () => { + const { checks, leaked } = await webhookChecks(durable({ gcAfterMs: 300 })); + for (const id of [ + 'sep-9999-ttl-long-grant-retained', + 'sep-9999-ttl-no-expiry-persisted', + 'sep-9999-ttl-no-expiry-gc-terminated' + ]) { + expect( + checks.get(id)?.status, + `${id}: ${checks.get(id)?.errorMessage}` + ).toBe('SUCCESS'); + } + expect(leaked).toEqual([]); + }); + + test('losing the no-expiry subscription on restart fails the persisted row', async () => { + const { checks } = await webhookChecks( + durable({ dropNoExpiryOnRestart: true, gcAfterMs: 300 }) + ); + const check = checks.get('sep-9999-ttl-no-expiry-persisted'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('never refreshes'); + expect(check?.errorMessage).toContain('`absent`'); + const gc = checks.get('sep-9999-ttl-no-expiry-gc-terminated'); + expect(gc?.details?.untestable, gc?.errorMessage).toBe(true); + }); + + test('losing a long finite grant on restart fails the retained row', async () => { + const { checks } = await webhookChecks( + durable({ dropFiniteOnRestart: true }) + ); + expect(checks.get('sep-9999-ttl-long-grant-retained')?.status).toBe( + 'FAILURE' + ); + }); + + test('never dropping a failing no-expiry subscription is a WARNING, since dropping is a MAY', async () => { + const { checks } = await webhookChecks(durable({})); + const check = checks.get('sep-9999-ttl-no-expiry-gc-terminated'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('MAY'); + }); + + test('a restart control that changes nothing leaves the rows untestable', async () => { + const { checks } = await webhookChecks(durable({ restartNoop: true })); + const check = checks.get('sep-9999-ttl-no-expiry-persisted'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('no evidence a restart happened'); + }); +}); diff --git a/src/scenarios/server/events/webhook.ts b/src/scenarios/server/events/webhook.ts index 9fb9409d..aca46139 100644 --- a/src/scenarios/server/events/webhook.ts +++ b/src/scenarios/server/events/webhook.ts @@ -35,6 +35,9 @@ import { EVENTS_EXTENSION_ID, EVENTS_CONTROL_SUBSCRIBE_AS, EVENTS_CONTROL_SUBSCRIPTION_EXISTS, + EVENTS_CONTROL_RESTART, + EVENTS_CONTROL_GENERATION, + EVENTS_CONTROL_SUBSCRIPTION_STATE, hasControl, askControl, extensionsOf, @@ -58,6 +61,16 @@ import { /** A callback URL that is syntactically valid and points nowhere in use. */ const CALLBACK_BASE = 'https://conformance.invalid/mcp-events'; +const DURABILITY_IDS = [ + 'sep-9999-ttl-long-grant-retained', + 'sep-9999-ttl-no-expiry-persisted', + 'sep-9999-ttl-no-expiry-gc-terminated' +] as const; +/** How long a restart may take to show up in the generation control. */ +const restartWaitMs = (): number => + Number(process.env.EVENTS_RESTART_WAIT_MS ?? 5000); +/** How long to watch a failing no-expiry subscription for a GC drop. */ +const gcWaitMs = (): number => Number(process.env.EVENTS_GC_WAIT_MS ?? 30000); /** A suggestion the server can plausibly grant, for the TTL rows. */ const TTL_SUGGESTION_MS = 3600_000; @@ -121,6 +134,11 @@ interface SubscribeResult { deliveryStatus?: unknown; } +/** How a subscription-state answer reads in a failure message. */ +function stateWord(state: string | undefined): string { + return state === undefined ? 'unreadable' : `reported \`${state}\``; +} + /** A Standard Webhooks secret: `whsec_` plus base64 of 32 random bytes. */ function freshSecret(): string { const bytes = new Uint8Array(32); @@ -209,9 +227,227 @@ export class EventsWebhookScenario implements ClientScenario { ); } - return await this.webhookChecks(conn, listed.descriptors, name, args); + const checks = await this.webhookChecks( + conn, + listed.descriptors, + name, + args + ); + // Last, because the restart ends this connection. + const durable = await this.durabilityChecks(ctx, conn, name, args); + return checks.map((c) => durable.get(c.id) ?? c); + } finally { + await conn.close().catch(() => undefined); + } + } + + /** + * The three TTL rows that need a restart. Only reachable through a fixture + * exposing the restart and subscription-state controls; without them the + * untestable rows from ttlChecks stand. + */ + private async durabilityChecks( + ctx: RunContext, + conn: Connection, + name: string, + args: Record + ): Promise> { + const out = new Map(); + const canRestart = await hasControl(conn, EVENTS_CONTROL_RESTART); + const canAsk = + (await hasControl(conn, EVENTS_CONTROL_SUBSCRIPTION_STATE)) && + (await hasControl(conn, EVENTS_CONTROL_GENERATION)); + if (!canRestart || !canAsk) return out; + + const stamp = Date.now(); + const subscribeWith = async ( + path: string, + ttlMs: number | null + ): Promise<{ id: string; url: string; grant: unknown } | undefined> => { + const url = `${CALLBACK_BASE}/${path}-${stamp}`; + try { + const result = await conn.request( + EVENTS_SUBSCRIBE_METHOD, + { + name, + arguments: args, + delivery: { mode: 'webhook', url, secret: freshSecret() }, + ttlMs + } + ); + return typeof result?.id === 'string' + ? { id: result.id, url, grant: result.refreshBefore } + : undefined; + } catch { + return undefined; + } + }; + + const noExpiry = await subscribeWith('ttl-null', null); + const long = await subscribeWith('ttl-long', 24 * 3600 * 1000); + const granted = noExpiry !== undefined && noExpiry.grant === null; + + const target = await askControl(conn, EVENTS_CONTROL_RESTART, {}); + // A stateful server ends the old session, so the connection has to be + // replaced; a stateless one keeps answering on the same connection, from + // the old build and then the new. The generation settles both. + let live: Connection = conn; + let opened: Connection | undefined; + const deadline = Date.now() + restartWaitMs(); + let restarted = false; + while (target !== undefined && Date.now() < deadline) { + const gen = await askControl(live, EVENTS_CONTROL_GENERATION, {}); + if (gen === target) { + restarted = true; + break; + } + if (gen === undefined) { + try { + await opened?.close().catch(() => undefined); + opened = await ctx.connect(); + await opened.discover(); + live = opened; + } catch { + // The new build may not be up yet; try again next tick. + } + } + await new Promise((resolve) => setTimeout(resolve, 100)); + } + if (!restarted) { + await opened?.close().catch(() => undefined); + for (const id of DURABILITY_IDS) { + out.set( + id, + untestableCheck( + id, + id, + id, + `The restart control answered ${describeValue(target)} but the generation control never reported it within ${restartWaitMs()}ms, so there is no evidence a restart happened.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + return out; + } + + const fresh = live; + try { + const state = async (id: string): Promise => + askControl(fresh, EVENTS_CONTROL_SUBSCRIPTION_STATE, { id }); + + if (long) { + const s = await state(long.id); + out.set( + 'sep-9999-ttl-long-grant-retained', + eventsCheck( + 'sep-9999-ttl-long-grant-retained', + 'A server granting long or no-expiry TTLs MUST retain subscriptions for the lifetime it granted, including across restarts.', + s === 'active' || s === 'suspended' ? 'SUCCESS' : 'FAILURE', + s === 'active' || s === 'suspended' + ? { details: { grant: long.grant, afterRestart: s } } + : { + errorMessage: `A subscription granted until ${String(long.grant)} was ${stateWord(s)} after a restart.` + } + ) + ); + } + + if (!granted) { + for (const id of [ + 'sep-9999-ttl-no-expiry-persisted', + 'sep-9999-ttl-no-expiry-gc-terminated' + ]) { + out.set( + id, + untestableCheck( + id, + id, + id, + `\`ttlMs: null\` was answered with refreshBefore ${describeValue(noExpiry?.grant)}, not null, so the server did not grant a no-expiry subscription. That is allowed; it only means these rows have nothing to grade.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + return out; + } + + const afterRestart = await state(noExpiry.id); + const persisted = + afterRestart === 'active' || afterRestart === 'suspended'; + out.set( + 'sep-9999-ttl-no-expiry-persisted', + eventsCheck( + 'sep-9999-ttl-no-expiry-persisted', + 'The server MUST persist no-expiry subscriptions across restarts, because a client that never refreshes will never detect (or repair) a silently dropped one.', + persisted ? 'SUCCESS' : 'FAILURE', + persisted + ? { details: { afterRestart } } + : { + errorMessage: `A no-expiry subscription was ${stateWord(afterRestart)} after a restart. A client that never refreshes has no way to find that out.` + } + ) + ); + + // The callback origin never resolves, so every delivery fails; a server + // that GCs no-expiry subscriptions should drop this one. + const gcDescription = + 'The server MAY drop a no-expiry subscription after sustained delivery failure (server-defined window), and SHOULD attempt a `terminated` envelope when it does.'; + let dropped = false; + const gcDeadline = Date.now() + gcWaitMs(); + while (persisted && Date.now() < gcDeadline) { + if ((await state(noExpiry.id)) === 'absent') { + dropped = true; + break; + } + await new Promise((resolve) => setTimeout(resolve, 500)); + } + out.set( + 'sep-9999-ttl-no-expiry-gc-terminated', + !persisted + ? untestableCheck( + 'sep-9999-ttl-no-expiry-gc-terminated', + 'sep-9999-ttl-no-expiry-gc-terminated', + gcDescription, + 'The no-expiry subscription was already gone after the restart, so there was nothing left for failure-based GC to drop.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + : dropped + ? eventsCheck( + 'sep-9999-ttl-no-expiry-gc-terminated', + gcDescription, + 'SUCCESS', + { + details: { + note: 'Dropped after sustained delivery failure. The terminated envelope goes to the placeholder callback origin, which the harness cannot receive at, so only the drop is observed.' + } + } + ) + : eventsCheck( + 'sep-9999-ttl-no-expiry-gc-terminated', + gcDescription, + 'WARNING', + { + errorMessage: `A no-expiry subscription whose every delivery failed was still registered after ${gcWaitMs()}ms. Dropping it is a MAY, so this is not a violation; a server with a long GC window reads the same way.` + } + ) + ); + + for (const sub of [long, noExpiry]) { + if (!sub) continue; + await fresh + .request(EVENTS_UNSUBSCRIBE_METHOD, { + name, + arguments: args, + delivery: { mode: 'webhook', url: sub.url } + }) + .catch(() => undefined); + } + return out; } finally { - await conn.close(); + await opened?.close().catch(() => undefined); } } From 3089d622818ed0678f021d9ef4bd3d0d88649263 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 15:14:27 +0000 Subject: [PATCH 17/27] fix(events): recognise every consent path, not just the handshake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The document gives a server four ways to confirm a callback wants deliveries: the challenge handshake, a configured allowlist, prior out-of-band verification, or a receiver-published /.well-known/mcp-webhook-receiver.json. The suite only knew the first, so it reported a server that read the document and delivered without challenging as one that "delivered to a callback that was never asked to prove intent". mcpkit 1449 landed that path today, which would have turned the row red for a server satisfying it. The receiver now publishes the document and counts fetches, and consent counts as established by a challenge or by a fetch that precedes the first delivery. It is declared over one path prefix and no other, so the main delivery path exercises the document route while every probe path still takes the handshake: one run covers both rather than whichever the server prefers. EVENTS_RECEIVER_WELL_KNOWN=0 withholds it. The document is honoured only on an https origin, so against a loopback receiver the handshake decides everything, the same way the SSRF rows only mean something over a tunnel. Two rows come out of that. sep-9999-verification-no-raw-endpoint-responses was an unconditional SUCCESS whose details read "no endpoint response body was echoed back in any server error observed during this run" — nothing had observed anything, so it could not have caught a leak. It now grades the error from the wrong-challenge probe, where the receiver echoes a distinctive string, and reports untestable on a run where no endpoint failed. sep-9999-error-callback-endpoint-error, declared since the first commit and never emitted, is claimed off that same error: the -32015 code and whether data.reason is one of the documented lastError categories. Why the failure branch stays a failure when nothing is observable: the allowlist and out-of-band paths cannot apply to these callbacks, because the path is minted fresh for the run and no operator has ever seen it. 7 new controls, 31 in negative-delivery.test.ts. Against the previous scenario the well-known case fails with "expected 'FAILURE' to be 'SUCCESS'", which is the regression this prevents. --- .../server/events/negative-delivery.test.ts | 133 +++++++++ .../server/events/negative-fixture.ts | 75 ++++- src/scenarios/server/events/receiver.ts | 52 +++- .../server/events/webhook-delivery.ts | 275 +++++++++++++++--- 4 files changed, 489 insertions(+), 46 deletions(-) diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index a28b4dbf..6408f847 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -512,3 +512,136 @@ describe.concurrent('control envelopes', () => { TIMEOUT ); }); + +describe.concurrent('confirming intent without a handshake', () => { + // The document allows four consent paths and the harness can only see two of + // them. Before this, a server that read the receiver's well-known document and + // delivered without a challenge failed the rule it had just satisfied. + test( + 'a server that reads the well-known document passes without challenging', + async () => { + const checks = await deliveryChecks( + delivering({ verifyViaWellKnown: true, verify: false }) + ); + const check = checks.get( + 'sep-9999-verification-required-before-delivery' + ); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.via).toBe( + '/.well-known/mcp-webhook-receiver.json' + ); + // The challenge rows say why they were not exercised, and how to force it. + const echo = checks.get('sep-9999-verification-challenge-echo'); + expect(echo?.details?.untestable).toBe(true); + expect(echo?.errorMessage).toContain('EVENTS_RECEIVER_WELL_KNOWN=0'); + }, + TIMEOUT + ); + + test( + 'delivering with neither a challenge nor a document read still fails', + async () => { + const checks = await deliveryChecks(delivering({ verify: false })); + const check = checks.get( + 'sep-9999-verification-required-before-delivery' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('third party'); + }, + TIMEOUT + ); +}); + +describe.concurrent('the -32015 error-table row', () => { + test( + 'a categorised challenge failure passes, and an uncategorised one warns', + async () => { + const ok = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + const good = ok.get('sep-9999-error-callback-endpoint-error'); + expect(good?.status).toBe('SUCCESS'); + expect(good?.details?.reason).toBe('challenge_failed'); + + const vague = await deliveryChecks( + delivering({ + synchronousVerification: true, + challengeFailureReason: 'it did not work' + }) + ); + const check = vague.get('sep-9999-error-callback-endpoint-error'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('not one of the documented'); + }, + TIMEOUT * 2 + ); + + test( + 'the wrong code for a failed callback fails the row', + async () => { + const checks = await deliveryChecks( + delivering({ + synchronousVerification: true, + challengeFailureCode: -32603 + }) + ); + const check = checks.get('sep-9999-error-callback-endpoint-error'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32015 CallbackEndpointError'); + }, + TIMEOUT + ); + + test( + 'a run where no callback ever fails reports the row untestable', + async () => { + const checks = await deliveryChecks(delivering()); + const check = checks.get('sep-9999-error-callback-endpoint-error'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('never provoked'); + }, + TIMEOUT + ); +}); + +describe.concurrent('leaking the endpoint response', () => { + // Retires an unconditional SUCCESS: the row used to pass on a run where + // nothing had failed, so it could never have caught a leak. + test( + 'echoing the endpoint body back in the error fails', + async () => { + const checks = await deliveryChecks( + delivering({ + synchronousVerification: true, + challengeFailureLeaksBody: true + }) + ); + const check = checks.get( + 'sep-9999-verification-no-raw-endpoint-responses' + ); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('reflection'); + }, + TIMEOUT + ); + + test( + 'reporting only the category passes, and a quiet run is untestable', + async () => { + const ok = await deliveryChecks( + delivering({ synchronousVerification: true }) + ); + expect( + ok.get('sep-9999-verification-no-raw-endpoint-responses')?.status + ).toBe('SUCCESS'); + + const quiet = await deliveryChecks(delivering()); + const check = quiet.get( + 'sep-9999-verification-no-raw-endpoint-responses' + ); + expect(check?.details?.untestable).toBe(true); + expect(check?.status).toBe('WARNING'); + }, + TIMEOUT * 2 + ); +}); diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index 38fad74b..4adcf1b8 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -109,6 +109,8 @@ export interface StreamBehaviour { maxConcurrent?: number; /** Open a stream for any name, including one the catalog does not serve. */ acceptAnyName?: boolean; + /** Name the quota in `data.limit` when refusing past `maxConcurrent`. */ + capLimitName?: string; } const CONFORMANT_STREAM: Required< @@ -198,6 +200,20 @@ export interface DeliveryBehaviour { synchronousVerification?: boolean; /** With `synchronousVerification`, accept the subscription anyway. */ ignoreFailedEcho?: boolean; + /** + * Confirm intent by fetching the callback origin's + * `/.well-known/mcp-webhook-receiver.json` instead of running the handshake, + * which is the fourth consent path the document allows. A document covering + * the callback path means no challenge POST; a 404 means no consent, and this + * fixture then delivers anyway, which is the case the rule exists to catch. + */ + verifyViaWellKnown?: boolean; + /** Code to answer when the handshake fails, where the table says -32015. */ + challengeFailureCode?: number; + /** `data.reason` for that failure, where the table wants a category. */ + challengeFailureReason?: string; + /** Echo the endpoint's own response body back in the failure message. */ + challengeFailureLeaksBody?: boolean; /** Deliver an event at all. */ sendEvent?: boolean; /** Deliver the event before the verification envelope. */ @@ -624,12 +640,24 @@ export async function startEventsFixture( held.at = held.at ?? Date.now(); } - if (opts.delivery?.synchronousVerification && typeof url === 'string') { + if (opts.delivery?.verifyViaWellKnown && typeof url === 'string') { + await fetchReceiverWellKnown(url); + } else if ( + opts.delivery?.synchronousVerification && + typeof url === 'string' + ) { const echoed = await challengeCallback(url, delivery.secret, id); - if (!echoed && !opts.delivery.ignoreFailedEcho) { - fail(-32015, 'endpoint verification failed', { - reason: 'challenge_failed' - }); + if (!echoed.ok && !opts.delivery.ignoreFailedEcho) { + const leak = opts.delivery.challengeFailureLeaksBody + ? `: endpoint answered ${echoed.body}` + : ''; + fail( + opts.delivery.challengeFailureCode ?? -32015, + `endpoint verification failed${leak}`, + { + reason: opts.delivery.challengeFailureReason ?? 'challenge_failed' + } + ); return; } } @@ -676,7 +704,11 @@ export async function startEventsFixture( return; } if (liveStreams >= behaviour.maxConcurrent) { - fail(-32013, 'ResourceExhausted: too many subscriptions'); + fail( + -32013, + 'ResourceExhausted: too many subscriptions', + behaviour.capLimitName ? { limit: behaviour.capLimitName } : undefined + ); return; } if (behaviour.answerJson) { @@ -861,11 +893,29 @@ const RETRY_GAP_MS = 1100; * one signed `verification` POST, and true only for a 2xx whose body echoes the * nonce. No retries and no redirects, the same as a delivery. */ +async function fetchReceiverWellKnown(callbackUrl: string): Promise { + try { + const origin = new URL(callbackUrl).origin; + const res = await fetch(`${origin}/.well-known/mcp-webhook-receiver.json`); + if (!res.ok) { + await res.text(); + return false; + } + const doc: unknown = await res.json(); + const prefixes = + isRecord(doc) && Array.isArray(doc.receivers) ? doc.receivers : []; + const path = new URL(callbackUrl).pathname; + return prefixes.some((p) => typeof p === 'string' && path.startsWith(p)); + } catch { + return false; + } +} + async function challengeCallback( url: string, secret: unknown, subscriptionId: string -): Promise { +): Promise<{ ok: boolean; body: string }> { const key = typeof secret === 'string' && secret.startsWith('whsec_') ? Buffer.from(secret.slice('whsec_'.length), 'base64') @@ -889,11 +939,14 @@ async function challengeCallback( redirect: 'manual' }); const text = await res.text(); - if (res.status < 200 || res.status >= 300) return false; + if (res.status < 200 || res.status >= 300) return { ok: false, body: text }; const reply: unknown = JSON.parse(text); - return isRecord(reply) && reply.challenge === challenge; - } catch { - return false; + return { + ok: isRecord(reply) && reply.challenge === challenge, + body: text + }; + } catch (err) { + return { ok: false, body: String(err) }; } } diff --git a/src/scenarios/server/events/receiver.ts b/src/scenarios/server/events/receiver.ts index 6c851eaa..1d034d56 100644 --- a/src/scenarios/server/events/receiver.ts +++ b/src/scenarios/server/events/receiver.ts @@ -12,6 +12,13 @@ * the challenge and accepts, one that redirects, one that refuses permanently, * and one that fails a few times before accepting. * + * The receiver can also publish `/.well-known/mcp-webhook-receiver.json`, which + * is the fourth way the document lets a server confirm intent: an origin that + * serves it has declared consent for the path prefixes it names, and no + * challenge POST is needed. `publishWellKnown` turns it on and `wellKnownFetches` + * counts the GETs, which is how the scenario tells that path apart from a server + * that simply skipped verification. + * * Every path answers the verification challenge first, whatever its behaviour, * because a probe exists to misbehave on deliveries. A 410 probe that also * refused the challenge would never get subscribed by a server that verifies @@ -45,6 +52,19 @@ export type PathBehaviour = | { kind: 'fail-then-accept'; failures: number; status: number } | { kind: 'wrong-challenge' }; +/** + * What the `wrong-challenge` path echoes instead of the nonce. + * + * Distinctive on purpose: the scenario looks for this string in whatever error + * the server reports afterwards, because the document says a failed handshake + * surfaces as a category and never as the endpoint's own response body. + */ +export const WRONG_CHALLENGE_ECHO = 'not-the-nonce'; + +/** Where a receiver declares which of its paths accept MCP deliveries. */ +export const RECEIVER_WELL_KNOWN_PATH = + '/.well-known/mcp-webhook-receiver.json'; + export interface Receiver { /** Base URL of the receiver, e.g. `http://127.0.0.1:53211`. */ readonly url: string; @@ -53,6 +73,14 @@ export interface Receiver { on(path: string): ReceivedDelivery[]; /** Set how a path answers. Unknown paths accept. */ behave(path: string, behaviour: PathBehaviour): void; + /** + * Serve the well-known document, declaring `prefixes` as consenting paths. + * Until this is called the path answers 404, which is what a receiver that + * cannot publish same-origin content looks like. + */ + publishWellKnown(prefixes: string[]): void; + /** How many times the well-known document has been fetched. */ + wellKnownFetches(): number; /** Resolve once a delivery on `path` matches, or undefined at the deadline. */ waitFor( path: string, @@ -67,6 +95,8 @@ export async function startReceiver(host = '127.0.0.1'): Promise { const behaviours = new Map(); const failureCounts = new Map(); const startedAt = Date.now(); + let wellKnown: string[] | undefined; + let wellKnownGets = 0; const server = http.createServer((req, res) => { const chunks: Buffer[] = []; @@ -84,6 +114,20 @@ export async function startReceiver(host = '127.0.0.1'): Promise { // Not JSON, which is itself something the scenario grades. } + // The well-known document is not a delivery, so it is answered before the + // per-path behaviours and recorded only as a fetch count. + if (path === RECEIVER_WELL_KNOWN_PATH) { + if (!wellKnown) { + res.writeHead(404, { 'content-type': 'application/json' }); + res.end('{}'); + return; + } + wellKnownGets += 1; + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ receivers: wellKnown })); + return; + } + const behaviour = behaviours.get(path) ?? { kind: 'accept' }; const respond = (status: number, body?: string): void => { deliveries.push({ @@ -133,7 +177,7 @@ export async function startReceiver(host = '127.0.0.1'): Promise { respond(413); return; case 'wrong-challenge': - respond(200, JSON.stringify({ challenge: 'not-the-nonce' })); + respond(200, JSON.stringify({ challenge: WRONG_CHALLENGE_ECHO })); return; case 'fail-then-accept': { const seen = failureCounts.get(path) ?? 0; @@ -171,6 +215,12 @@ export async function startReceiver(host = '127.0.0.1'): Promise { behave(path, behaviour) { behaviours.set(path, behaviour); }, + publishWellKnown(prefixes) { + wellKnown = prefixes; + }, + wellKnownFetches() { + return wellKnownGets; + }, async waitFor(path, predicate, timeoutMs) { const deadline = Date.now() + timeoutMs; for (;;) { diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index 80cf6845..bf9e15d9 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -49,6 +49,8 @@ import { minimalArguments } from './helpers'; import { + RECEIVER_WELL_KNOWN_PATH, + WRONG_CHALLENGE_ECHO, startReceiver, type ReceivedDelivery, type Receiver @@ -68,6 +70,39 @@ const SETTLE_MS = Number(process.env.EVENTS_DELIVERY_SETTLE_MS ?? 5000); /** A public base URL forwarding to this harness, when one exists. */ const PUBLIC_BASE = process.env.EVENTS_WEBHOOK_CALLBACK_BASE; +/** + * Whether the receiver publishes `/.well-known/mcp-webhook-receiver.json`. + * + * The document is the fourth way the spec lets a server confirm a callback's + * intent, and an origin that serves it needs no challenge POST. Publishing it by + * default costs nothing against a server that only implements the handshake, and + * stops one that implements the document from failing a rule it satisfies. Set + * `EVENTS_RECEIVER_WELL_KNOWN=0` to withhold it and force the handshake path. + */ +const PUBLISH_WELL_KNOWN = process.env.EVENTS_RECEIVER_WELL_KNOWN !== '0'; + +/** + * The one path prefix the well-known document declares. + * + * Deliberately not `/`: the probe paths sit outside it so they still take the + * handshake. The document is also honoured only on an `https` origin, so against + * a loopback receiver this path is never taken and the handshake decides + * everything, the same way the SSRF rows only mean something over a tunnel. + */ +const WELL_KNOWN_PREFIX = '/wk/'; + +/** + * The `lastError` categories the document names for `-32015`. A server may have + * more, so an unlisted one warns rather than fails: the rule is that the reason + * is a category, not a raw response. + */ +const LAST_ERROR_CATEGORIES = [ + 'challenge_failed', + 'connection_refused', + 'timeout', + 'tls_error' +]; + /** 256 KiB, the body-size ceiling the document asks servers to respect. */ const BODY_CEILING_BYTES = 256 * 1024; @@ -111,11 +146,20 @@ const ENVELOPE_IDS = [ 'sep-9999-envelope-terminated' ] as const; +/** + * The error-table row this scenario claims. `-32015` is webhook-only and the + * wrong-challenge probe is the only place the suite provokes it, so the row is + * graded there and listed here for the same reason the others are: a path that + * bails early must still report it, rather than emit one row fewer. + */ +const ERROR_IDS = ['sep-9999-error-callback-endpoint-error'] as const; + const ALL_IDS = [ ...DELIVERY_IDS, ...VERIFICATION_IDS, ...SSRF_IDS, - ...ENVELOPE_IDS + ...ENVELOPE_IDS, + ...ERROR_IDS ]; function untestableAll( @@ -225,6 +269,12 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } receiver = await startReceiver(PUBLIC_BASE ? '0.0.0.0' : '127.0.0.1'); + // Only the main delivery path is declared, never the probe paths. A + // server that reads the document verifies that one without a challenge, + // and still has to handshake for everything under a prefix the document + // does not name, so one run exercises both consent paths instead of + // whichever the server happens to prefer. + if (PUBLISH_WELL_KNOWN) receiver.publishWellKnown([WELL_KNOWN_PREFIX]); return await this.deliveryChecks(conn, receiver, name, args); } finally { await receiver?.close(); @@ -245,7 +295,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { args: Record ): Promise { const checks: ConformanceCheck[] = []; - const path = `/hook-${Date.now()}`; + const path = `${WELL_KNOWN_PREFIX}hook-${Date.now()}`; const url = this.callbackFor(receiver, path); const secret = freshSecret(); @@ -295,7 +345,12 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { checks.push( ...this.ssrfRefusedChecks(refused), ...untestableAll( - [...DELIVERY_IDS, ...VERIFICATION_IDS, ...ENVELOPE_IDS], + [ + ...DELIVERY_IDS, + ...VERIFICATION_IDS, + ...ENVELOPE_IDS, + ...ERROR_IDS + ], `The server refused a loopback callback (${refused.code} ${refused.message}), which is what the SSRF rules ask of it. Grading delivery needs a routable callback: set EVENTS_WEBHOOK_CALLBACK_BASE to a public https URL forwarding to this harness.` ) ); @@ -361,7 +416,12 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { if (!first) { checks.push( ...untestableAll( - [...DELIVERY_IDS, ...VERIFICATION_IDS, ...ENVELOPE_IDS].filter( + [ + ...DELIVERY_IDS, + ...VERIFICATION_IDS, + ...ENVELOPE_IDS, + ...ERROR_IDS + ].filter( (id) => id !== 'sep-9999-delivery-status-last-error-category' ), `Nothing arrived at ${url} within ${DELIVERY_WAIT_MS}ms of subscribing, so no delivery could be graded.` @@ -389,12 +449,29 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { return dedupe(checks); } - checks.push(...this.verificationChecks(all, subscriptionId)); + checks.push( + ...this.verificationChecks( + all, + subscriptionId, + receiver.wellKnownFetches() + ) + ); + // The residual rows come last because one of them grades what the server + // said when the handshake failed, which only the probe below produces. + let endpointFailure: JsonRpcError | undefined; if (all.some(isVerificationEnvelope)) { - checks.push( - await this.verificationFailureCheck(receiver, subscribe, release) + const failed = await this.verificationFailureCheck( + receiver, + subscribe, + release ); + checks.push(failed.check); + endpointFailure = failed.serverError; } + checks.push( + ...this.verificationResidualChecks(subscriptionId, endpointFailure) + ); + checks.push(this.callbackEndpointErrorCheck(endpointFailure)); checks.push(...this.transportChecks(all, subscriptionId)); checks.push(...this.signatureChecks(all, secret.bytes)); checks.push(...this.envelopeChecks(all)); @@ -472,12 +549,47 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { /** The challenge handshake that must precede any event delivery. */ private verificationChecks( all: ReceivedDelivery[], - subscriptionId: unknown + subscriptionId: unknown, + wellKnownFetches: number ): ConformanceCheck[] { const out: ConformanceCheck[] = []; const verification = all.find(isVerificationEnvelope); const events = all.filter((d) => typeof d.json?.eventId === 'string'); + // The document names four ways to confirm intent, and two of them are + // invisible from here: a server-configured allowlist and prior out-of-band + // verification. Neither can apply to these callbacks, because the path is + // minted fresh for the run and no operator has ever seen it, so a server + // that delivers to it has either handshaken, read the well-known document, + // or skipped consent. That is what keeps the failure below honest. + if (!verification && wellKnownFetches > 0) { + out.push( + eventsCheck( + 'sep-9999-verification-required-before-delivery', + "A server MUST NOT begin delivering to a callback URL until the endpoint's intent to receive deliveries is confirmed, by one of: a verification handshake, a server-configured allowlist, prior out-of-band verification, or a receiver-published well-known document.", + 'SUCCESS', + { + details: { + via: RECEIVER_WELL_KNOWN_PATH, + fetches: wellKnownFetches, + note: 'The origin published its consent, so no challenge POST is required.' + } + } + ) + ); + out.push( + ...untestableAll( + [ + 'sep-9999-verification-challenge-echo', + 'sep-9999-verification-failure-error' + ], + `The server confirmed intent by fetching ${RECEIVER_WELL_KNOWN_PATH} and sent no challenge, so the echo path was not exercised. Re-run with EVENTS_RECEIVER_WELL_KNOWN=0 to withhold the document and force the handshake.`, + 'WARNING' + ) + ); + return out; + } + if (!verification) { out.push( eventsCheck( @@ -539,6 +651,20 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ); } + return out; + } + + /** + * The verification rows that do not depend on which consent path the server + * took, kept in one place so every branch above emits the same row set. A + * branch that emitted fewer would read as a shorter suite rather than as a + * prerequisite that was missing. + */ + private verificationResidualChecks( + subscriptionId: unknown, + endpointFailure?: JsonRpcError + ): ConformanceCheck[] { + const out: ConformanceCheck[] = []; out.push( untestableCheck( 'sep-9999-verification-uses-ssrf-hardened-path', @@ -560,20 +686,91 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { 'WARNING' ) ); - out.push( - eventsCheck( - 'sep-9999-verification-no-raw-endpoint-responses', - 'Failures surface only via the `lastError` category `challenge_failed`, never raw endpoint responses.', - 'SUCCESS', - { + out.push(this.noRawEndpointResponsesCheck(subscriptionId, endpointFailure)); + return out; + } + + /** + * The `-32015` row from the error table, graded off the same probe as + * sep-9999-verification-failure-error rather than a second failing callback. + * That row grades the rule; this one grades the code and the `data.reason` + * category the table requires of it. + */ + private callbackEndpointErrorCheck( + endpointFailure?: JsonRpcError + ): ConformanceCheck { + const id = 'sep-9999-error-callback-endpoint-error'; + const description = + '`-32015 CallbackEndpointError` — a client-supplied callback endpoint failed verification or could not be reached (webhook mode only). `data.reason` is one of the `lastError` categories.'; + if (!endpointFailure) { + return untestableCheck( + id, + id, + description, + 'No callback failed verification or connection during this run, so the code was never provoked. A server that implements the verification handshake answers it for the wrong-challenge probe.', + [EVENTS_SPEC_REF] + ); + } + const reason = isObject(endpointFailure.data) + ? endpointFailure.data.reason + : undefined; + if (endpointFailure.code !== EVENTS_CALLBACK_ENDPOINT_ERROR) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `A callback that failed verification answered ${endpointFailure.code} ${endpointFailure.message}, where the table names ${EVENTS_CALLBACK_ENDPOINT_ERROR} CallbackEndpointError.`, + details: { code: endpointFailure.code, data: endpointFailure.data } + }); + } + return LAST_ERROR_CATEGORIES.includes(String(reason)) + ? eventsCheck(id, description, 'SUCCESS', { details: { - note: 'No endpoint response body was echoed back in any server error observed during this run.', - subscriptionId + code: endpointFailure.code, + reason, + gradedBy: 'sep-9999-verification-failure-error' } - } - ) - ); - return out; + }) + : eventsCheck(id, description, 'WARNING', { + errorMessage: `\`${EVENTS_CALLBACK_ENDPOINT_ERROR}\` carried \`data.reason\` ${describeValue(reason)}, which is not one of the documented \`lastError\` categories (${LAST_ERROR_CATEGORIES.join(', ')}).`, + details: { reason } + }); + } + + /** + * Whether a failed handshake leaked the endpoint's own response. + * + * Only gradeable when something actually failed, which is why it waits for the + * wrong-challenge probe rather than passing on the strength of a quiet run. + * The receiver echoes a distinctive string, so finding it in the server's + * error is proof the body was passed through to the subscriber. + */ + private noRawEndpointResponsesCheck( + subscriptionId: unknown, + endpointFailure?: JsonRpcError + ): ConformanceCheck { + const id = 'sep-9999-verification-no-raw-endpoint-responses'; + const description = + 'Failures surface only via the `lastError` category `challenge_failed`, never raw endpoint responses.'; + if (!endpointFailure) { + return untestableCheck( + id, + id, + description, + 'No endpoint failed verification during this run, so nothing could have carried its response body back. The wrong-challenge probe supplies one against a server that implements the handshake.', + [EVENTS_SPEC_REF], + 'WARNING' + ); + } + const reported = JSON.stringify({ + message: endpointFailure.message, + data: endpointFailure.data + }); + return reported.includes(WRONG_CHALLENGE_ECHO) + ? eventsCheck(id, description, 'FAILURE', { + errorMessage: `The error for a failed handshake carried the endpoint's own response (${JSON.stringify(WRONG_CHALLENGE_ECHO)}). A subscriber learns what an arbitrary third-party URL answered, which is the reflection the category exists to avoid.`, + details: { reported } + }) + : eventsCheck(id, description, 'SUCCESS', { + details: { subscriptionId, code: endpointFailure.code } + }); } /** POST, content type, and the headers every delivery must carry. */ @@ -938,7 +1135,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { url: string ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, release: (url: string) => Promise - ): Promise { + ): Promise<{ check: ConformanceCheck; serverError?: JsonRpcError }> { const id = 'sep-9999-verification-failure-error'; const description = 'A reachable endpoint that fails to echo yields `-32015 CallbackEndpointError` with `data.reason: "challenge_failed"`.'; @@ -954,14 +1151,20 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { code === EVENTS_CALLBACK_ENDPOINT_ERROR && reason === 'challenge_failed' ) { - return eventsCheck(id, description, 'SUCCESS', { - details: { code, reason } - }); + return { + check: eventsCheck(id, description, 'SUCCESS', { + details: { code, reason } + }), + serverError: probe.error + }; } - return eventsCheck(id, description, 'FAILURE', { - errorMessage: `Subscribing a callback that echoed the wrong nonce answered ${code} ${message} with data.reason ${describeValue(reason)}, expected ${EVENTS_CALLBACK_ENDPOINT_ERROR} with "challenge_failed".`, - details: { code, data } - }); + return { + check: eventsCheck(id, description, 'FAILURE', { + errorMessage: `Subscribing a callback that echoed the wrong nonce answered ${code} ${message} with data.reason ${describeValue(reason)}, expected ${EVENTS_CALLBACK_ENDPOINT_ERROR} with "challenge_failed".`, + details: { code, data } + }), + serverError: probe.error + }; } try { @@ -971,13 +1174,17 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { DELIVERY_WAIT_MS ); if (delivered) { - return eventsCheck(id, description, 'FAILURE', { - errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce, and an event was then delivered to it. The document has this refused with ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" from events/subscribe, and no delivery to an endpoint that did not consent.` - }); + return { + check: eventsCheck(id, description, 'FAILURE', { + errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce, and an event was then delivered to it. The document has this refused with ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" from events/subscribe, and no delivery to an endpoint that did not consent.` + }) + }; } - return eventsCheck(id, description, 'WARNING', { - errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce. Nothing was delivered to it, so the endpoint is safe, but the subscriber was never told: the document returns ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" synchronously from events/subscribe.` - }); + return { + check: eventsCheck(id, description, 'WARNING', { + errorMessage: `The subscribe succeeded although the callback echoed the wrong nonce. Nothing was delivered to it, so the endpoint is safe, but the subscriber was never told: the document returns ${EVENTS_CALLBACK_ENDPOINT_ERROR} "challenge_failed" synchronously from events/subscribe.` + }) + }; } finally { await release(url); } From 5aceffa371fc1d84d5aa3f60a3df362ee11da34e Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 15:14:42 +0000 Subject: [PATCH 18/27] feat(events): claim four rows the probes already reach MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four declared rows emitted nothing while the request that would grade each was already being sent. Claiming them costs no new round trips. sep-9999-fallback-method-not-found — what a server that does not offer the extension answers. discovery.ts already probes events/list on a server that declares nothing and keys its SKIP off -32601, so the one run that ever meets such a server now grades the rule instead of folding it into skipAll. A server answering some other code fails it: a client cannot tell "no such extension" from a real failure. Everything else still skips, because every other events rule is genuinely inapplicable there. sep-9999-capability-empty-settings — `{}` declares support with no list-change notifications. Graded off the declared value, and inapplicable rather than failed when the server declared settings of its own, since the rule says what an empty object means and nothing about a populated one. sep-9999-error-invalid-params and -resource-exhausted — the error-table rows for -32602 and -32013. The first rides the poll invalid-arguments probe, the way sep-9999-subscribe-url-https-required already rides its enforcement probe, with details.gradedBy naming where the verdict came from. The second grades the three concurrent streams in push.ts: a refusal for a quota must name the quota in data.limit, and a server that reaches no limit reports untestable rather than passing. 117 of 135 rows emitted before this and the commit before it; 122 now. Of the 13 left, seven wait on a control that can change a catalog mid-run, two on a second credential, and sep-9999-error-forbidden on a principal the harness can be refused as. 5 new controls. One existing assertion changes: the "skips the suite" case now asserts the fallback row passes and everything else skips, which adds an assertion rather than relaxing one. No assertion weakened. --- src/scenarios/server/events/discovery.ts | 93 ++++++++++++++++-- .../server/events/negative-push.test.ts | 23 +++++ src/scenarios/server/events/negative.test.ts | 47 ++++++++- src/scenarios/server/events/poll.ts | 97 +++++++++++++------ src/scenarios/server/events/push.ts | 59 +++++++++-- 5 files changed, 276 insertions(+), 43 deletions(-) diff --git a/src/scenarios/server/events/discovery.ts b/src/scenarios/server/events/discovery.ts index 6829d8f0..0737550e 100644 --- a/src/scenarios/server/events/discovery.ts +++ b/src/scenarios/server/events/discovery.ts @@ -56,7 +56,8 @@ import { const CAPABILITY_IDS = [ 'sep-9999-capability-events-object', - 'sep-9999-capability-list-changed-flag' + 'sep-9999-capability-list-changed-flag', + 'sep-9999-capability-empty-settings' ] as const; const LIST_IDS = [ @@ -75,9 +76,13 @@ const DESCRIPTOR_IDS = [ const ERROR_IDS = [ 'sep-9999-error-not-found', - 'sep-9999-error-server-range' + 'sep-9999-error-server-range', + 'sep-9999-fallback-method-not-found' ] as const; +const FALLBACK_DESCRIPTION = + 'A server that does not offer the extension answers any `events/*` request with `-32601 MethodNotFound` (standard JSON-RPC).'; + const ALL_IDS = [ ...CAPABILITY_IDS, ...LIST_IDS, @@ -97,7 +102,7 @@ export class EventsDiscoveryScenario implements ClientScenario { readonly source = { extensionId: EVENTS_EXTENSION_ID } as const; description = `MCP Events: capability declaration, \`events/list\` enumeration, and the error-code contract. -**Methods**: \`events/list\` (mandatory for a server declaring \`capabilities.events\`), \`events/poll\` (probed only for its error path) +**Methods**: \`events/list\` (mandatory for a server declaring the extension), \`events/poll\` (probed only for its error path) **Requirements covered** (each check carries a verbatim spec excerpt in src/seps/sep-9999.yaml): @@ -127,7 +132,7 @@ export class EventsDiscoveryScenario implements ClientScenario { // --- Capability ------------------------------------------------------ const capDescription = - 'Servers advertise event support in their capabilities as an object under `capabilities.events`.'; + "Events is declared through Extension Negotiation: the identifier appears as a key in the `extensions` field of capabilities, mapped to the extension's settings object."; const { declared, value } = await declaredEventsCapability(conn); if (!declared) { @@ -137,10 +142,24 @@ export class EventsDiscoveryScenario implements ClientScenario { // report it as a clean run. Distinguish the two by asking. const probe = await eventsListPage(conn); if ('error' in probe && probe.error.code === JSONRPC_METHOD_NOT_FOUND) { - return skipAll( + // Not an events server, so every rule about events is inapplicable — + // except the one that says what such a server answers, which it just + // did. Grading it here is the only place the suite ever meets a server + // that does not offer the extension. + const skipped = skipAll( 'Server does not declare the `events` capability and does not implement `events/list`; the extension is optional.' - ); + ).filter((c) => c.id !== 'sep-9999-fallback-method-not-found'); + return [ + ...skipped, + eventsCheck( + 'sep-9999-fallback-method-not-found', + FALLBACK_DESCRIPTION, + 'SUCCESS', + { details: { code: probe.error.code } } + ) + ]; } + checks.push(this.fallbackCheck(probe)); checks.push( eventsCheck( 'sep-9999-capability-events-object', @@ -185,7 +204,7 @@ export class EventsDiscoveryScenario implements ClientScenario { { errorMessage: declared ? 'Server did not declare `listChanged`; the flag is optional and its absence means the notification is not advertised.' - : 'Server declared no `capabilities.events` object for the flag to sit in; see sep-9999-capability-events-object.' + : 'Server declared no settings object for the flag to sit in; see sep-9999-capability-events-object.' } ) ); @@ -205,13 +224,15 @@ export class EventsDiscoveryScenario implements ClientScenario { 'The `listChanged` flag advertises that the server sends `notifications/events/list_changed`.', 'FAILURE', { - errorMessage: `\`capabilities.events.listChanged\` is ${describeValue(listChanged)}, expected a boolean.`, + errorMessage: `\`listChanged\` in the extension's settings object is ${describeValue(listChanged)}, expected a boolean.`, details: { listChanged } } ) ); } + checks.push(this.emptySettingsCheck(declared, value)); + // --- events/list ----------------------------------------------------- const firstPage = await eventsListPage(conn); if ('error' in firstPage) { @@ -283,6 +304,62 @@ export class EventsDiscoveryScenario implements ClientScenario { * base protocol, so the only thing this check can establish is that * `events/list` participates in the scheme at all. */ + /** + * What a server that does not serve events answers to an `events/*` request. + * + * Only reachable on a server that declares nothing, since one that offers the + * extension has no business answering `-32601`. The skip path above grades the + * clean case; this one grades a server that answered something else, which + * leaves a client unable to tell "no such extension" from a real failure. + */ + private fallbackCheck( + probe: Awaited> + ): ConformanceCheck { + const id = 'sep-9999-fallback-method-not-found'; + if (!('error' in probe)) { + return eventsCheck(id, FALLBACK_DESCRIPTION, 'SKIPPED', { + errorMessage: `\`${EVENTS_LIST_METHOD}\` returned a catalog, so this server does offer the extension and the fallback does not apply to it.` + }); + } + return eventsCheck(id, FALLBACK_DESCRIPTION, 'FAILURE', { + errorMessage: `A server declaring no events capability answered \`${EVENTS_LIST_METHOD}\` with ${probe.error.code} ${probe.error.message}, where a server that does not offer the extension answers ${JSONRPC_METHOD_NOT_FOUND} MethodNotFound.`, + details: { code: probe.error.code } + }); + } + + /** + * `{}` as the settings object: support declared, no list-change notifications. + * + * Inapplicable rather than failed when the server declared settings of its + * own, because the rule describes what an empty object means and says nothing + * about a populated one. + */ + private emptySettingsCheck( + declared: boolean, + value: unknown + ): ConformanceCheck { + const id = 'sep-9999-capability-empty-settings'; + const description = + 'An empty settings object declares event support with no list-change notifications.'; + if (!declared || !isObject(value)) { + return eventsCheck(id, description, 'SKIPPED', { + errorMessage: + 'No settings object was declared, so there is no empty-object case to grade; see sep-9999-capability-events-object.' + }); + } + const keys = Object.keys(value); + if (keys.length > 0) { + return eventsCheck(id, description, 'SKIPPED', { + errorMessage: `The server declared settings (${keys.join(', ')}), so the empty-object case does not apply to it.` + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { + note: 'Empty settings, and the catalog is still served; sep-9999-list-implemented grades that half.' + } + }); + } + private async paginationCheck( conn: Awaited>, nextCursor: unknown diff --git a/src/scenarios/server/events/negative-push.test.ts b/src/scenarios/server/events/negative-push.test.ts index 0bf65039..b78bb64d 100644 --- a/src/scenarios/server/events/negative-push.test.ts +++ b/src/scenarios/server/events/negative-push.test.ts @@ -293,6 +293,29 @@ describe.concurrent('the heartbeat', () => { }); }); +describe.concurrent('the -32013 error-table row', () => { + test('a cap that names its limit passes, and one that does not warns', async () => { + const named = await pushChecks( + pushFixture({ maxConcurrent: 1, capLimitName: 'subscriptions' }) + ); + const ok = named.get('sep-9999-error-resource-exhausted'); + expect(ok?.status).toBe('SUCCESS'); + expect(ok?.details?.limit).toBe('subscriptions'); + + const vague = await pushChecks(pushFixture({ maxConcurrent: 1 })); + const check = vague.get('sep-9999-error-resource-exhausted'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('which quota it hit'); + }); + + test('a server that refuses nothing reports the row untestable', async () => { + const checks = await pushChecks(pushFixture()); + const check = checks.get('sep-9999-error-resource-exhausted'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('never provoked'); + }); +}); + describe.concurrent('concurrency and cancellation', () => { // The cap kitchen-sink applies to streams, which the document exempts them // from: the first stream confirms and the other two are refused -32013. diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts index b790e778..3dd2682d 100644 --- a/src/scenarios/server/events/negative.test.ts +++ b/src/scenarios/server/events/negative.test.ts @@ -74,15 +74,45 @@ describe('events capability declaration', () => { expect(check?.errorMessage).toContain('a boolean'); }); - test('a server that declares nothing and serves nothing skips the suite', async () => { + test('a server that declares nothing and serves nothing skips all but the fallback row', async () => { const checks = await checksFor(discovery(), { listError: { code: -32601, message: 'Method not found' } }); + // Every events rule is inapplicable to a server that does not do events, + // except the one saying what such a server answers. It answered it. + expect(checks.get('sep-9999-fallback-method-not-found')?.status).toBe( + 'SUCCESS' + ); for (const check of checks.values()) { - expect(check.status).toBe('SKIPPED'); + if (check.id === 'sep-9999-fallback-method-not-found') continue; + expect(check.status, check.id).toBe('SKIPPED'); } }); + test('answering something other than -32601 fails the fallback row', async () => { + const checks = await checksFor(discovery(), { + listError: { code: -32000, message: 'nope' } + }); + const check = checks.get('sep-9999-fallback-method-not-found'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('-32601 MethodNotFound'); + }); + + test('empty settings pass, and declared settings make the rule inapplicable', async () => { + const empty = await checksFor(discovery(), { + ...CONFORMANT, + capability: {} + }); + expect(empty.get('sep-9999-capability-empty-settings')?.status).toBe( + 'SUCCESS' + ); + + const populated = await checksFor(discovery(), CONFORMANT); + const check = populated.get('sep-9999-capability-empty-settings'); + expect(check?.status).toBe('SKIPPED'); + expect(check?.errorMessage).toContain('listChanged'); + }); + // The case mcpkit is actually in: events/list answers, but nothing is // declared, so a client that reads capabilities first never calls it. A // plain SKIP here would report that as a clean run. @@ -325,6 +355,19 @@ describe('events/poll error contract', () => { ); }); + test('the -32602 error-table row carries the same verdict as the rule', async () => { + const ok = await checksFor(poll(), CONFORMANT); + const row = ok.get('sep-9999-error-invalid-params'); + expect(row?.status).toBe('SUCCESS'); + expect(row?.details?.gradedBy).toBe('sep-9999-poll-invalid-arguments'); + + const broken = await checksFor(poll(), { + ...CONFORMANT, + invalidArgsCode: -32011 + }); + expect(broken.get('sep-9999-error-invalid-params')?.status).toBe('FAILURE'); + }); + test('wrong-typed arguments answering -32011 fails the invalid-params check', async () => { const ok = await checksFor(poll(), CONFORMANT); expect(ok.get('sep-9999-poll-invalid-arguments')?.status).toBe('SUCCESS'); diff --git a/src/scenarios/server/events/poll.ts b/src/scenarios/server/events/poll.ts index 17d8e3e5..180d0ade 100644 --- a/src/scenarios/server/events/poll.ts +++ b/src/scenarios/server/events/poll.ts @@ -98,7 +98,42 @@ const CURSOR_IDS = [ 'sep-9999-truncated-false-when-no-replay' ] as const; -const ALL_IDS = [...POLL_IDS, ...OCCURRENCE_IDS, ...CURSOR_IDS]; +/** + * Pair the `-32602` rule row with the error-table row for the same code. + * + * One probe, two rows: the first says arguments that violate `inputSchema` are + * rejected, the second says the code for a statically invalid request is + * `-32602`. A server gets the same verdict on both because there is nothing to + * tell apart, and `details.gradedBy` says where it came from. + */ +function withErrorCodeRow(ruleCheck: ConformanceCheck): ConformanceCheck[] { + return [ + ruleCheck, + eventsCheck( + 'sep-9999-error-invalid-params', + "`-32602 InvalidParams` — request is statically invalid: arguments don't match the event's inputSchema, the callback `delivery.url` is malformed or non-`https`, or `delivery.secret` is not a valid `whsec_` value.", + ruleCheck.status, + { + errorMessage: ruleCheck.errorMessage, + details: { + gradedBy: 'sep-9999-poll-invalid-arguments', + untestable: ruleCheck.details?.untestable + } + } + ) + ]; +} + +/** + * The error-table row this scenario claims. `-32602` has three provoking cases + * in the document and the suite already fires one of them here, so the row is + * graded off that probe rather than a fourth request, the way + * sep-9999-subscribe-url-https-required rides its enforcement probe in + * events-webhook. + */ +const ERROR_IDS = ['sep-9999-error-invalid-params'] as const; + +const ALL_IDS = [...POLL_IDS, ...OCCURRENCE_IDS, ...CURSOR_IDS, ...ERROR_IDS]; /** Milliseconds of replay to request when probing the `maxAgeMs` floor. */ const MAX_AGE_PROBE_MS = 300_000; @@ -1101,7 +1136,9 @@ export class EventsPollScenario implements ClientScenario { ); // Invalid arguments. Only probeable when the schema constrains something. - out.push(await this.invalidArgumentsCheck(conn, descriptors, name, args)); + out.push( + ...(await this.invalidArgumentsCheck(conn, descriptors, name, args)) + ); // A delivery mode the event type does not offer. out.push(await this.unsupportedModeCheck(conn, descriptors)); @@ -1121,7 +1158,7 @@ export class EventsPollScenario implements ClientScenario { descriptors: EventDescriptor[], name: string, _args: Record - ): Promise { + ): Promise { const id = 'sep-9999-poll-invalid-arguments'; const description = "A poll whose `arguments` do not match the event's `inputSchema` returns `-32602 InvalidParams`."; @@ -1142,13 +1179,15 @@ export class EventsPollScenario implements ClientScenario { : undefined; if (!typed) { - return untestableCheck( - id, - id, - description, - `Event type \`${name}\` declares no typed \`inputSchema\` property, so no argument value can be known-invalid against it.`, - [EVENTS_SPEC_REF], - 'FAILURE' + return withErrorCodeRow( + untestableCheck( + id, + id, + description, + `Event type \`${name}\` declares no typed \`inputSchema\` property, so no argument value can be known-invalid against it.`, + [EVENTS_SPEC_REF], + 'FAILURE' + ) ); } @@ -1164,25 +1203,29 @@ export class EventsPollScenario implements ClientScenario { }); if (!('error' in probe)) { - return eventsCheck(id, description, 'FAILURE', { - errorMessage: `A poll sending \`${prop}: ${JSON.stringify(wrongValue)}\` against a declared \`${propType}\` returned a result instead of ${JSONRPC_INVALID_PARAMS} InvalidParams.`, - details: { property: prop, declaredType: propType, sent: wrongValue } - }); + return withErrorCodeRow( + eventsCheck(id, description, 'FAILURE', { + errorMessage: `A poll sending \`${prop}: ${JSON.stringify(wrongValue)}\` against a declared \`${propType}\` returned a result instead of ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { property: prop, declaredType: propType, sent: wrongValue } + }) + ); } - return probe.error.code === JSONRPC_INVALID_PARAMS - ? eventsCheck(id, description, 'SUCCESS', { - details: { property: prop, declaredType: propType } - }) - : eventsCheck(id, description, 'FAILURE', { - errorMessage: `Arguments violating \`inputSchema\` answered ${probe.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, - details: { - property: prop, - declaredType: propType, - code: probe.error.code, - message: probe.error.message - } - }); + return withErrorCodeRow( + probe.error.code === JSONRPC_INVALID_PARAMS + ? eventsCheck(id, description, 'SUCCESS', { + details: { property: prop, declaredType: propType } + }) + : eventsCheck(id, description, 'FAILURE', { + errorMessage: `Arguments violating \`inputSchema\` answered ${probe.error.code}, expected ${JSONRPC_INVALID_PARAMS} InvalidParams.`, + details: { + property: prop, + declaredType: propType, + code: probe.error.code, + message: probe.error.message + } + }) + ); } /** Poll an event type whose `delivery` omits `poll`. */ diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts index cc6fcd07..6322ea58 100644 --- a/src/scenarios/server/events/push.ts +++ b/src/scenarios/server/events/push.ts @@ -53,6 +53,7 @@ import { EVENTS_STREAM_METHOD, EVENTS_TERMINATED_NOTIFICATION, EVENTS_NOT_FOUND, + EVENTS_RESOURCE_EXHAUSTED, JSONRPC_METHOD_NOT_FOUND, SUBSCRIPTION_ID_META, describeValue, @@ -100,6 +101,14 @@ const STREAM_IDS = [ 'sep-9999-stream-carries-only-event-notifications' ] as const; +/** + * The error-table row this scenario claims. `-32013` is only reached when a + * server actually refuses for a quota, and the concurrency probe is the one + * place the suite asks for more than one of anything, so the row is graded there + * and reported untestable on a server that never hits a limit. + */ +const ERROR_IDS = ['sep-9999-error-resource-exhausted'] as const; + function untestableAll( ids: readonly string[], reason: string, @@ -111,7 +120,7 @@ function untestableAll( } function skipAll(reason: string): ConformanceCheck[] { - return STREAM_IDS.map((id) => + return [...STREAM_IDS, ...ERROR_IDS].map((id) => eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) ); } @@ -154,7 +163,7 @@ export class EventsPushScenario implements ClientScenario { ); } return untestableAll( - STREAM_IDS, + [...STREAM_IDS, ...ERROR_IDS], `\`events/list\` failed (${listed.error.code} ${listed.error.message}), so no push-capable event type could be discovered. See the events-discovery scenario.` ); } @@ -163,7 +172,7 @@ export class EventsPushScenario implements ClientScenario { const name = target ? descriptorName(target) : undefined; if (!target || !name) { return untestableAll( - STREAM_IDS, + [...STREAM_IDS, ...ERROR_IDS], listed.descriptors.length === 0 ? '`events/list` returned an empty catalog, so no push-capable event type could be exercised.' : 'No event type advertises `push` delivery, so `events/stream` could not be exercised. Push is optional per event type.' @@ -184,7 +193,7 @@ export class EventsPushScenario implements ClientScenario { const args = minimalArguments(target); if (args === undefined) { return untestableAll( - STREAM_IDS, + [...STREAM_IDS, ...ERROR_IDS], `Event type \`${name}\` declares required \`inputSchema\` properties the harness cannot satisfy from the schema, so no stream could be opened.` ); } @@ -235,7 +244,9 @@ export class EventsPushScenario implements ClientScenario { ); checks.push( ...untestableAll( - STREAM_IDS.filter((id) => id !== 'sep-9999-stream-implemented'), + [...STREAM_IDS, ...ERROR_IDS].filter( + (id) => id !== 'sep-9999-stream-implemented' + ), `No stream was opened for \`${name}\`, so nothing on it could be observed.` ) ); @@ -939,13 +950,49 @@ export class EventsPushScenario implements ClientScenario { })) } } - ) + ), + this.resourceExhaustedCheck(sessions.map((s) => s.error)) ]; } finally { await Promise.all(sessions.map((s) => s.cancel())); } } + /** + * The `-32013` row from the error table. + * + * A conformant server reaches no limit here, so this usually reports + * untestable. It grades when one refuses, which is worth claiming rather than + * leaving to the concurrency row: that row says streams are exempt from the + * cap, this one says a refusal for a quota names the quota. + */ + private resourceExhaustedCheck( + errors: (StreamSession['error'] | undefined)[] + ): ConformanceCheck { + const id = 'sep-9999-error-resource-exhausted'; + const description = + '`-32013 ResourceExhausted` — a server-imposed limit or quota was reached. `data.limit` names it (e.g. `"subscriptions"`).'; + const hit = errors.find((e) => e?.code === EVENTS_RESOURCE_EXHAUSTED); + if (!hit) { + return untestableCheck( + id, + id, + description, + 'No request in this run was refused for a server-imposed limit, so the code was never provoked. A server that caps concurrent subscriptions answers it for the third stream above.', + [EVENTS_SPEC_REF] + ); + } + const limit = isObject(hit.data) ? hit.data.limit : undefined; + return typeof limit === 'string' && limit.length > 0 + ? eventsCheck(id, description, 'SUCCESS', { + details: { limit, message: hit.message } + }) + : eventsCheck(id, description, 'WARNING', { + errorMessage: `\`${EVENTS_RESOURCE_EXHAUSTED}\` carried \`data.limit\` ${describeValue(limit)}. Without it a client cannot tell which quota it hit, so it cannot know what to stop doing.`, + details: { data: hit.data, message: hit.message } + }); + } + /** An invalid subscription answers an error and opens no stream. */ private async errorBeforeOpenChecks( ctx: RunContext, From 3051ea275cf873c1eb8741bbb38dd5c7e1a09db0 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 15:14:55 +0000 Subject: [PATCH 19/27] docs(events): correct the counts, and name the PR the capability rows track The extensions-map change added four rows and restated two, and the header's arithmetic did not follow: it still said 131 declared, 117 of 131 emitted, and "the 14 rows nothing emits yet". It is 135, 122 and 13. It also said "see the tracking note at the head of this file" and there was no such note. There is now, as a banner: the six rows that quote text only PR 7 has, why the suite is ahead of the document, and to re-point spec_source at main when it merges. spec_source and spec_url named `main`, where a reader who fetched it found text contradicting those rows; both now name PR 7's head. Seven strings still told implementers the capability lives at `capabilities.events`, including the description attached to sep-9999-capability-events-object itself, so one check carried two contradictory statements and the one a reader of a failing run sees was the wrong one. The survivors were exactly the strings no control asserts on, which is also where to add an assertion. The note in helpers.ts keeps the old spelling deliberately, because it is explaining the history. declaredEventsCapability now calls extensionsOf rather than inlining the same lookup, which is what that helper's doc comment promises. --- src/scenarios/index.ts | 3 +- src/scenarios/server/events/helpers.ts | 2 +- src/seps/sep-9999.yaml | 85 ++++++++++++++++++-------- src/types.ts | 10 +-- 4 files changed, 68 insertions(+), 32 deletions(-) diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 82899e06..73c38633 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -237,7 +237,8 @@ const allClientScenariosList: ClientScenario[] = [ new SkillsEnumerationScenario(), new SkillsManifestScenario(), - // MCP Events. Fixture-dependent (needs a server declaring `capabilities.events`); + // MCP Events. Fixture-dependent (needs a server declaring the extension under + // `capabilities.extensions`); // each scenario SKIPs cleanly when the capability is not declared. new EventsDiscoveryScenario(), new EventsPollScenario(), diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index f7b1999e..969b9285 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -181,7 +181,7 @@ export async function declaredEventsCapability( ): Promise<{ declared: boolean; value: unknown }> { const discovered = await conn.discover(); const caps = (discovered.capabilities as Record) ?? {}; - const exts = (caps.extensions as Record) ?? {}; + const exts = extensionsOf(caps); if (!(EVENTS_EXTENSION_ID in exts)) return { declared: false, value: undefined }; return { declared: true, value: exts[EVENTS_EXTENSION_ID] }; diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 52a99629..31d87435 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -1,5 +1,23 @@ -# spec_source: modelcontextprotocol/experimental-ext-triggers-events@main docs/design-sketch-proposal.md -# extracted: 2026-09-15 +# spec_source: modelcontextprotocol/experimental-ext-triggers-events@refs/pull/7/head docs/design-sketch-proposal.md +# extracted: 2026-09-15, capability rows re-extracted 2026-09-23 +# +# ############################################################################ +# # SIX ROWS TRACK AN UNMERGED PR. Upstream PR 7 moves the capability # +# # declaration into Extension Negotiation, and `main` still says top-level. # +# ############################################################################ +# +# The rows are sep-9999-capability-events-object, -list-changed-flag, +# -empty-settings, -client-declaration, -list-changed-gated and +# sep-9999-fallback-method-not-found. `spec_source` therefore names PR 7's head +# rather than `main`, because a reader who fetches `main` finds text that +# contradicts them. +# +# Why the suite is ahead of the document: mcpkit followed the sketch and +# declared top-level; metronome, written by the sketch's author, used the +# extensions map. Asked rather than guessed, 2026-09-22, and the document was +# the thing that was wrong. Same posture as the SEP-2350 scenarios against +# modelcontextprotocol#481: a red row here means the head moved, not that an +# implementation regressed. Re-point `spec_source` at `main` when PR 7 merges. # # ############################################################################ # # 9999 IS A PLACEHOLDER. MCP Events has no SEP number yet. # @@ -45,7 +63,7 @@ # `spec_url` names the document itself and rows carry no per-row `url`. # Expect one more pass once the text becomes a spec PR and gains anchors. # -# coverage: 131 declared checks plus 30 excluded rows. +# coverage: 135 declared checks plus 30 excluded rows. # # A keyword sweep of the source finds 144 RFC 2119 keyword occurrences across # 119 sentences: 34 MUST, 9 MUST NOT, 3 REQUIRED, 56 SHOULD, 7 SHOULD NOT @@ -53,8 +71,8 @@ # and SHOULD NOT inside SHOULD, as the handoff table does, the same sweep # reads as 43 / 3 / 63; the two counts agree. # -# Only 46 of the declared rows quote a sentence carrying a keyword. The other -# 85 are shape requirements the document states declaratively — the +# Only 47 of the declared rows quote a sentence carrying a keyword. The other +# 88 are shape requirements the document states declaratively — the # `EventOccurrence` field table, the error-code table, the `events/list` # descriptor fields, the response payloads. They are normative and directly # testable, they are simply not written with a keyword, which is why the row @@ -100,25 +118,34 @@ # # backing_scenarios: server ClientScenarios under src/scenarios/server/events/ # emit the check IDs below (a row is "tested" once a scenario emits its ID; -# see src/traceability/). All five now exist and together emit 117 of the 131 +# see src/traceability/). All five now exist and together emit 122 of the 135 # rows: -# discovery.ts (events-discovery), 12 rows — the two capability rows, the -# two sep-9999-list-* rows, the six sep-9999-descriptor-* rows, and -# sep-9999-error-not-found plus sep-9999-error-server-range, both graded -# off one unknown-name probe. -# poll.ts (events-poll), 33 rows — the sep-9999-poll-* rows, the +# discovery.ts (events-discovery), 14 rows — the three capability rows, the +# two sep-9999-list-* rows, the six sep-9999-descriptor-* rows, +# sep-9999-error-not-found plus sep-9999-error-server-range (both graded +# off one unknown-name probe), and sep-9999-fallback-method-not-found, +# which only a server that does not offer the extension can answer. +# poll.ts (events-poll), 35 rows — the sep-9999-poll-* rows, the # sep-9999-occurrence-* rows, the sep-9999-cursor-* / # sep-9999-max-age-* / sep-9999-truncated-* rows reachable through poll, -# and sep-9999-removal-poll-not-found, the poll leg of the removal rules. -# push.ts (events-push), 17 rows — the sep-9999-stream-* rows. It holds a +# sep-9999-removal-poll-not-found (the poll leg of the removal rules), and +# sep-9999-error-invalid-params, which rides the same probe as +# sep-9999-poll-invalid-arguments. +# push.ts (events-push), 18 rows — the sep-9999-stream-* rows plus +# sep-9999-error-resource-exhausted, graded when a server refuses one of +# the three concurrent streams for a quota. It holds a # real SSE stream open through stream.ts, because conn.request() resolves # on the response for its id and a push stream withholds that until the # subscription ends. # webhook.ts (events-webhook), 27 rows — sep-9999-subscribe-*, # sep-9999-ttl-*, sep-9999-unsubscribe-* and sep-9999-error-unsupported. -# webhook-delivery.ts (events-webhook-delivery), 28 rows — -# sep-9999-delivery-*, sep-9999-verification-*, sep-9999-ssrf-* and -# sep-9999-envelope-*, graded from an HTTP receiver the harness runs. +# webhook-delivery.ts (events-webhook-delivery), 29 rows — +# sep-9999-delivery-*, sep-9999-verification-*, sep-9999-ssrf-*, +# sep-9999-envelope-* and sep-9999-error-callback-endpoint-error, graded +# from an HTTP receiver the harness runs. The receiver publishes +# /.well-known/mcp-webhook-receiver.json over one path prefix and no other, +# so a server that takes the document path and one that handshakes are both +# exercised in a single run; EVENTS_RECEIVER_WELL_KNOWN=0 withholds it. # # negative controls: src/scenarios/server/events/negative*.test.ts, one file per # scenario over a shared fixture in negative-fixture.ts. A green run against @@ -133,17 +160,23 @@ # report untestable against both implementations, so the controls are the only # evidence those checks work at all. # -# The 14 rows nothing emits yet, and what each needs: +# The 13 rows nothing emits yet, and what each needs: # sep-9999-schema-evolution-additive, sep-9999-breaking-change-new-name, # sep-9999-list-changed-notification and the four remaining # sep-9999-removal-* rows need a server whose catalog can be made to change # mid-run. Neither fixture can, so these wait on a driver that can tell the # server under test to add, alter or drop an event type. -# sep-9999-error-invalid-params, -forbidden, -resource-exhausted and -# -callback-endpoint-error are the error-code table rows. Each is currently -# graded through the specific rule that provokes it rather than in its own -# right; kitchen-sink's -32013 subscription cap and the -32015 verification -# failure are both reachable and worth claiming in a follow-up. +# sep-9999-error-forbidden is the last error-code table row with no probe: +# it needs a principal the harness can be refused as. The other three are +# claimed off the rule that provokes each, with details.gradedBy naming it: +# -32602 off the poll invalid-arguments probe, -32013 off the concurrent +# streams, -32015 off the wrong-challenge callback. +# sep-9999-capability-list-changed-gated needs a server that declares +# listChanged false and then sends the notification anyway; nothing in the +# suite watches for an unsolicited notification yet. +# sep-9999-capability-client-declaration is a MAY about what a client +# declares, so no server-side scenario can emit it. Declared for now; the +# honest place for it is `excluded:` beside the other MAY rows. # sep-9999-authz-subscribe-time and -delivery-time-reverify need two # principals with different permissions. # sep-9999-payload-minimality is a SHOULD about payload content, which @@ -238,8 +271,8 @@ # picking a side. **The document was wrong**: extension negotiation is where # it belongs, and upstream PR 7 moves the sketch there. metronome was right # all along and mcpkit's conformance was an artifact of following a document -# that did not yet say what its author meant. The rows above now track PR 7, -# which is unmerged; see the tracking note at the head of this file. +# that did not yet say what its author meant. The rows above track PR 7; the +# banner at the head of this file names them and says what to do on merge. # sep-9999-poll-invalid-arguments — arguments that violate `inputSchema` # answered -32603 Internal error, where the document requires -32602 # InvalidParams. Read as a framework default surfacing rather than a @@ -380,7 +413,9 @@ # sep-9999-delivery-signature-formula. # sep: 9999 -spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/main/docs/design-sketch-proposal.md +# Points at PR 7's head, not `main`: six capability rows quote text that only +# exists there. See the banner at the head of this file. +spec_url: https://github.com/modelcontextprotocol/experimental-ext-triggers-events/blob/refs/pull/7/head/docs/design-sketch-proposal.md requirements: # === Capability Declaration === - check: sep-9999-capability-events-object diff --git a/src/types.ts b/src/types.ts index 970f97e4..20c058b8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -104,11 +104,11 @@ export const EXTENSION_IDS = [ 'io.modelcontextprotocol/auth/wif', 'io.modelcontextprotocol/tasks', 'io.modelcontextprotocol/skills', - // MCP Events declares its capability top-level as `capabilities.events`, - // not inside `capabilities.extensions`. This id is a suite-selection key - // only — it keeps the Events scenarios off the `--spec-version` timeline - // (see `matchesSpecVersion`), and is never a path into the capability - // object. See src/scenarios/server/events/helpers.ts. + // MCP Events. This id does double duty: it keeps the Events scenarios off the + // `--spec-version` timeline (see `matchesSpecVersion`), and it is the key the + // capability lives under in `capabilities.extensions`. Extension negotiation + // arrived in 2026-07-28, so no earlier protocol version can declare it. + // See src/scenarios/server/events/helpers.ts. 'io.modelcontextprotocol/events' ] as const; export type ExtensionId = (typeof EXTENSION_IDS)[number]; From e18b608a60631efdff392b8706c8a2846c93625a Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 15:20:12 +0000 Subject: [PATCH 20/27] docs(events): keep a check slug out of the file that does not emit it The comment introducing poll's error-table row named the webhook precedent by its exact slug, which makes `grep sep-9999-subscribe-url-https-required` report two scenarios emitting it when only one does. It now names the rule in prose. The header's count for poll.ts followed the same miscount and read 35 where the file emits 34; the suite total of 122 was right either way. --- src/scenarios/server/events/poll.ts | 7 ++++--- src/seps/sep-9999.yaml | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/scenarios/server/events/poll.ts b/src/scenarios/server/events/poll.ts index 180d0ade..8a9c8ff2 100644 --- a/src/scenarios/server/events/poll.ts +++ b/src/scenarios/server/events/poll.ts @@ -127,9 +127,10 @@ function withErrorCodeRow(ruleCheck: ConformanceCheck): ConformanceCheck[] { /** * The error-table row this scenario claims. `-32602` has three provoking cases * in the document and the suite already fires one of them here, so the row is - * graded off that probe rather than a fourth request, the way - * sep-9999-subscribe-url-https-required rides its enforcement probe in - * events-webhook. + * graded off that probe rather than a fourth request, the way the + * https-required row rides its enforcement probe in events-webhook. That + * precedent is named in prose rather than by its slug, so grepping a check id + * still finds only the scenario that emits it. */ const ERROR_IDS = ['sep-9999-error-invalid-params'] as const; diff --git a/src/seps/sep-9999.yaml b/src/seps/sep-9999.yaml index 31d87435..8be8a01c 100644 --- a/src/seps/sep-9999.yaml +++ b/src/seps/sep-9999.yaml @@ -125,7 +125,7 @@ # sep-9999-error-not-found plus sep-9999-error-server-range (both graded # off one unknown-name probe), and sep-9999-fallback-method-not-found, # which only a server that does not offer the extension can answer. -# poll.ts (events-poll), 35 rows — the sep-9999-poll-* rows, the +# poll.ts (events-poll), 34 rows — the sep-9999-poll-* rows, the # sep-9999-occurrence-* rows, the sep-9999-cursor-* / # sep-9999-max-age-* / sep-9999-truncated-* rows reachable through poll, # sep-9999-removal-poll-not-found (the poll leg of the removal rules), and From 51f85d7e1aafaa4ec337a9e92bb49f82017349e9 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 15:35:34 +0000 Subject: [PATCH 21/27] fix(events): emit every row on every path, which a real run caught Measuring against kitchen-sink at mcpkit d7624d68 found two rows going missing rather than reporting a missing prerequisite, both in paths I had just added to. sep-9999-fallback-method-not-found was only emitted when the server declared nothing, so against every real events server it vanished. It is now SKIPPED there with the reason: a server that declares the extension is not a server that does not offer it, so the rule cannot apply. events-webhook-delivery emitted 27 rows when it refused a loopback callback and 29 when it delivered. The refusal answers two of the four SSRF rows by itself; the other two need a delivery to revalidate and a redirect to refuse, and were simply absent. Both now report untestable there. This is the defect the suite's own untestable policy exists to prevent: a row that appears on one path and not another makes two servers' pass counts incomparable, which is the whole point of a fixed denominator. Controls for both, asserting the row is present and inapplicable rather than absent. --- src/scenarios/server/events/discovery.ts | 14 ++++++++++++-- .../server/events/negative-delivery.test.ts | 10 ++++++++++ src/scenarios/server/events/negative.test.ts | 10 ++++++++++ src/scenarios/server/events/webhook-delivery.ts | 6 +++++- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/src/scenarios/server/events/discovery.ts b/src/scenarios/server/events/discovery.ts index 0737550e..b6b9e9ad 100644 --- a/src/scenarios/server/events/discovery.ts +++ b/src/scenarios/server/events/discovery.ts @@ -134,6 +134,9 @@ export class EventsDiscoveryScenario implements ClientScenario { const capDescription = "Events is declared through Extension Negotiation: the identifier appears as a key in the `extensions` field of capabilities, mapped to the extension's settings object."; const { declared, value } = await declaredEventsCapability(conn); + /** Set only when the server declared nothing, which is the one case the + * fallback rule applies to. */ + let undeclaredProbe: Awaited> | undefined; if (!declared) { // An undeclared optional capability is normally a SKIP. It is not one @@ -159,7 +162,7 @@ export class EventsDiscoveryScenario implements ClientScenario { ) ]; } - checks.push(this.fallbackCheck(probe)); + undeclaredProbe = probe; checks.push( eventsCheck( 'sep-9999-capability-events-object', @@ -232,6 +235,7 @@ export class EventsDiscoveryScenario implements ClientScenario { } checks.push(this.emptySettingsCheck(declared, value)); + checks.push(this.fallbackCheck(undeclaredProbe)); // --- events/list ----------------------------------------------------- const firstPage = await eventsListPage(conn); @@ -313,9 +317,15 @@ export class EventsDiscoveryScenario implements ClientScenario { * leaves a client unable to tell "no such extension" from a real failure. */ private fallbackCheck( - probe: Awaited> + probe: Awaited> | undefined ): ConformanceCheck { const id = 'sep-9999-fallback-method-not-found'; + if (!probe) { + return eventsCheck(id, FALLBACK_DESCRIPTION, 'SKIPPED', { + errorMessage: + 'The server declares the extension, so it is not a server that does not offer it and the fallback does not apply. Only a server declaring nothing can answer this rule.' + }); + } if (!('error' in probe)) { return eventsCheck(id, FALLBACK_DESCRIPTION, 'SKIPPED', { errorMessage: `\`${EVENTS_LIST_METHOD}\` returned a catalog, so this server does offer the extension and the fallback does not apply to it.` diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index 6408f847..17f8ee45 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -159,6 +159,16 @@ describe.concurrent('a server that delivers to loopback', () => { const signature = checks.get('sep-9999-delivery-signature-formula'); expect(signature?.details?.untestable).toBe(true); expect(signature?.errorMessage).toContain('EVENTS_WEBHOOK_CALLBACK_BASE'); + // The refusal answers two SSRF rows and not the other two, which still + // have to be reported: a row that appears only on the delivering path + // makes the two runs' counts incomparable. + for (const id of [ + 'sep-9999-ssrf-no-redirects', + 'sep-9999-ssrf-validate-at-delivery-time' + ]) { + expect(checks.get(id), id).toBeDefined(); + expect(checks.get(id)?.details?.untestable, id).toBe(true); + } }, TIMEOUT ); diff --git a/src/scenarios/server/events/negative.test.ts b/src/scenarios/server/events/negative.test.ts index 3dd2682d..577702f8 100644 --- a/src/scenarios/server/events/negative.test.ts +++ b/src/scenarios/server/events/negative.test.ts @@ -89,6 +89,16 @@ describe('events capability declaration', () => { } }); + test('the fallback row is emitted even when it cannot apply', async () => { + // A row that vanishes on some servers makes pass counts incomparable + // between them, which is why every path emits the whole set. + const checks = await checksFor(discovery(), CONFORMANT); + const check = checks.get('sep-9999-fallback-method-not-found'); + expect(check).toBeDefined(); + expect(check?.status).toBe('SKIPPED'); + expect(check?.errorMessage).toContain('declares the extension'); + }); + test('answering something other than -32601 fails the fallback row', async () => { const checks = await checksFor(discovery(), { listError: { code: -32000, message: 'nope' } diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index bf9e15d9..de8f913f 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -349,7 +349,11 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ...DELIVERY_IDS, ...VERIFICATION_IDS, ...ENVELOPE_IDS, - ...ERROR_IDS + ...ERROR_IDS, + // The two SSRF rows the refusal does not answer by itself: one + // needs a delivery to revalidate, the other a redirect to refuse. + 'sep-9999-ssrf-validate-at-delivery-time', + 'sep-9999-ssrf-no-redirects' ], `The server refused a loopback callback (${refused.code} ${refused.message}), which is what the SSRF rules ask of it. Grading delivery needs a routable callback: set EVENTS_WEBHOOK_CALLBACK_BASE to a public https URL forwarding to this harness.` ) From 07890dc54154f5b1b68345a5521ee5a2bf43a194 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 16:08:03 +0000 Subject: [PATCH 22/27] feat(events): grade delivery against a hardened server, one origin at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit events-webhook-delivery grades 29 rows and reaches 2 of them against a server with its SSRF guards on. Not a defect on either side: the harness must be the receiver, because the signature, the headers, the handshake and the retry cadence are only visible from the endpoint being POSTed to, so it listens on loopback — and a hardened server refuses a loopback callback, which is what two of those rows grade. Both cannot hold for one subscription. The way out is a fixture control, the same shape as the other eight: events_conformance_allow_callback_origin permits one named origin past the callback guards for the rest of the process. The order is what keeps the SSRF verdict honest. Subscribe with the guards on, grade the two SSRF rows against the server as configured, then lift for this receiver's origin and subscribe again, so signing, headers, retries, envelopes and verification grade off real deliveries. Nothing about hardening is concluded after the override. sep-9999-ssrf-validate-at-delivery-time now reports SKIPPED when the origin was permitted on purpose, because a delivery to a deliberately-allowed origin is evidence of nothing; proving revalidation still needs a hostname whose DNS answer changes between subscribe and delivery. The control does not exist yet: mcpkit 1457 has the contract, and this side sits behind hasControl, so a server without it reports untestable exactly as before and names both ways out, the tunnel and the control. 26 of the 27 rows become reachable when it lands. Two controls cover both paths. Writing them turned up the fixture refusing a permitted origin anyway, because its https check ran first — which also showed the "harden" flag I had added was redundant, since a fixture without acceptHttpUrl already refuses the harness's loopback callback. Gone, and the permitted set now bypasses that check the way the real control will. npm test: 751 passing. --- src/scenarios/server/events/helpers.ts | 18 +++++ .../server/events/negative-delivery.test.ts | 73 +++++++++++++++++++ .../server/events/negative-fixture.ts | 60 +++++++++++++-- .../server/events/webhook-delivery.ts | 59 ++++++++++++++- 4 files changed, 201 insertions(+), 9 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 969b9285..87bcda45 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -215,6 +215,24 @@ export const EVENTS_CONTROL_TERMINATE = 'events_conformance_terminate'; * only way to construct the two-tenant case the key-composition rule is about. */ export const EVENTS_CONTROL_SUBSCRIBE_AS = 'events_conformance_subscribe_as'; +/** + * Permits callbacks under one origin past the scheme and routability guards for + * the rest of the fixture process, so the harness can be delivered to. + * + * The harness necessarily listens on loopback, and a hardened server refuses a + * loopback callback — correctly, which is what the SSRF rows grade. Those two + * facts cannot both hold in one subscription, so without this the delivery rows + * are unreachable unless EVENTS_WEBHOOK_CALLBACK_BASE points at a public tunnel. + * + * Takes `{ origin }` and names one origin, not a blanket "allow private + * networks": the fixture stays hardened for every other callback, which is what + * lets the SSRF rows be graded first and then the delivery rows after. mcpkit's + * `--conformance-events` build already allowlists one origin this way for the + * events-webhook scenario, which is spec path (b) doing the same job. + */ +export const EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN = + 'events_conformance_allow_callback_origin'; + /** Reports whether a given principal's subscription is still registered. */ export const EVENTS_CONTROL_SUBSCRIPTION_EXISTS = 'events_conformance_subscription_exists'; diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index 17f8ee45..b2744328 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -655,3 +655,76 @@ describe.concurrent('leaking the endpoint response', () => { TIMEOUT * 2 ); }); + +describe.concurrent( + 'a hardened server the harness can still be delivered to', + () => { + // The harness listens on loopback and a hardened server refuses a loopback + // callback, correctly. Both facts cannot hold in one subscription, so without + // an override the delivery rows are unreachable on any server that enforces + // the rule the SSRF rows grade. + test( + 'the SSRF rows are graded first, then the guard is lifted for one origin', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: { allowCallbackOriginControl: true }, + delivery: {} + }); + + // Graded against the default configuration, before anything was lifted. + expect(checks.get('sep-9999-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + expect(checks.get('sep-9999-ssrf-reject-non-routable')?.status).toBe( + 'SUCCESS' + ); + + // And the behavioural rows now grade, which is the whole point. + for (const id of [ + 'sep-9999-delivery-post-json', + 'sep-9999-delivery-standard-webhooks-headers', + 'sep-9999-delivery-signature-formula', + 'sep-9999-verification-required-before-delivery', + 'sep-9999-envelope-type-discriminator' + ]) { + expect(checks.get(id)?.status, id).toBe('SUCCESS'); + } + + // A delivery to a deliberately-permitted origin proves nothing about + // delivery-time revalidation, so that row says so rather than passing. + const rebind = checks.get('sep-9999-ssrf-validate-at-delivery-time'); + expect(rebind?.status).toBe('SKIPPED'); + expect(rebind?.errorMessage).toContain('permitted through'); + }, + TIMEOUT * 2 + ); + + test( + 'without the control the rows stay untestable and name both ways out', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: {}, + delivery: {} + }); + expect(checks.get('sep-9999-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + const check = checks.get('sep-9999-delivery-signature-formula'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('EVENTS_WEBHOOK_CALLBACK_BASE'); + expect(check?.errorMessage).toContain( + 'events_conformance_allow_callback_origin' + ); + }, + TIMEOUT + ); + } +); diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index 4adcf1b8..5c072814 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -174,6 +174,15 @@ export interface SubscribeBehaviour { unsubscribeHeldCode?: number; /** Answer -32013 once this many subscriptions are live. */ maxSubscriptions?: number; + /** + * Expose `events_conformance_allow_callback_origin` as a tool, which permits + * one origin past the callback guards for the rest of the process. + * + * There is no separate "harden" switch: leaving `acceptHttpUrl` unset already + * refuses the harness's loopback http callback, which is what a server with + * its SSRF guards on does. + */ + allowCallbackOriginControl?: boolean; } /** @@ -347,6 +356,8 @@ export async function startEventsFixture( let generation = 1; /** Deliveries still in flight, so close() can settle rather than abandon. */ const inFlight = new Set>(); + /** Origins the callback guard has been asked to permit. */ + const permittedOrigins = new Set(); let mintedIds = 0; const queue = [...(opts.pollResponses ?? [pollResult()])]; const descriptors = opts.descriptors ?? [descriptor()]; @@ -387,15 +398,42 @@ export async function startEventsFixture( ); }; - if (opts.durability && method === 'tools/list') { + if (method === 'tools/list') { const obj = { type: 'object', properties: {} }; - send({ - tools: [ + const tools = []; + if (opts.durability) { + tools.push( { name: 'events_conformance_restart', inputSchema: obj }, { name: 'events_conformance_generation', inputSchema: obj }, { name: 'events_conformance_subscription_state', inputSchema: obj } - ] - }); + ); + } + if (opts.subscribe?.allowCallbackOriginControl) { + tools.push({ + name: 'events_conformance_allow_callback_origin', + inputSchema: obj + }); + } + if (tools.length > 0 || opts.durability) { + send({ tools }); + return; + } + } + + if ( + method === 'tools/call' && + params.name === 'events_conformance_allow_callback_origin' + ) { + const toolArgs = (params.arguments ?? {}) as Record; + if (typeof toolArgs.origin !== 'string') { + send({ + content: [{ type: 'text', text: 'origin required' }], + isError: true + }); + return; + } + permittedOrigins.add(toolArgs.origin); + send({ content: [{ type: 'text', text: toolArgs.origin }] }); return; } if (opts.durability && method === 'tools/call') { @@ -554,7 +592,19 @@ export async function startEventsFixture( fail(rejectionCode, `InvalidParams: \`delivery.secret\` (${problem})`); return; } + // A permitted origin bypasses the callback guards, which is what the real + // control does: one origin, both guards, for the rest of the process. + let callbackOrigin = ''; + try { + callbackOrigin = new URL(String(url)).origin; + } catch { + callbackOrigin = ''; + } + const permittedCallback = + callbackOrigin !== '' && permittedOrigins.has(callbackOrigin); + if ( + !permittedCallback && !behaviour.acceptHttpUrl && (typeof url !== 'string' || !url.startsWith('https://')) ) { diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index de8f913f..0d3eaef6 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -43,6 +43,9 @@ import { descriptorName, eventsCheck, eventsListAll, + EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN, + askControl, + hasControl, firstSupporting, isIso8601, isObject, @@ -335,15 +338,22 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } }; - const subscribed = await subscribe(url); + let subscribed = await subscribe(url); // --- The SSRF rows, which a loopback callback answers directly --------- const loopback = !PUBLIC_BASE; + let guardLifted = false; + if ('error' in subscribed && loopback) { + // Graded here, before anything is lifted, so the verdict is about the + // server as configured rather than as persuaded. + checks.push(...this.ssrfRefusedChecks(subscribed.error)); + guardLifted = await this.liftCallbackGuard(conn, receiver); + if (guardLifted) subscribed = await subscribe(url); + } if ('error' in subscribed) { const refused = subscribed.error; if (loopback) { checks.push( - ...this.ssrfRefusedChecks(refused), ...untestableAll( [ ...DELIVERY_IDS, @@ -355,7 +365,9 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { 'sep-9999-ssrf-validate-at-delivery-time', 'sep-9999-ssrf-no-redirects' ], - `The server refused a loopback callback (${refused.code} ${refused.message}), which is what the SSRF rules ask of it. Grading delivery needs a routable callback: set EVENTS_WEBHOOK_CALLBACK_BASE to a public https URL forwarding to this harness.` + guardLifted + ? `The server still refused ${url} after \`${EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN}\` was called for its origin (${refused.code} ${refused.message}), so nothing could be delivered.` + : `The server refused a loopback callback (${refused.code} ${refused.message}), which is what the SSRF rules ask of it. Grading delivery needs a callback it will accept: set EVENTS_WEBHOOK_CALLBACK_BASE to a public https URL forwarding to this harness, or expose the \`${EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN}\` control so this one origin is permitted after the SSRF rows are graded.` ) ); return dedupe(checks); @@ -393,8 +405,22 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ); const all = receiver.on(path); - if (loopback) { + if (loopback && !guardLifted) { checks.push(...this.ssrfDeliveredChecks(all.length > 0, url)); + } else if (loopback) { + // Already graded above, against the default configuration. Deliveries + // here happened because this origin was permitted on purpose, so they + // say nothing about the rule. + checks.push( + eventsCheck( + 'sep-9999-ssrf-validate-at-delivery-time', + 'To prevent DNS rebinding, validation MUST be performed at delivery time, not only at subscribe time.', + 'SKIPPED', + { + errorMessage: `This origin was permitted through \`${EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN}\`, so a delivery to it is not evidence either way. Proving revalidation needs a hostname whose DNS answer changes between subscribe and delivery.` + } + ) + ); } else { checks.push( ...untestableAll( @@ -487,6 +513,31 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } } + /** + * Ask the server to permit this receiver's origin, after the SSRF rows have + * been graded against its default configuration. + * + * Absent control means no override, which is the normal case and never an + * error: the delivery rows then report untestable and name both ways out. + * Returns whether the server acknowledged, not whether the next subscribe will + * succeed, because a server may decline for its own reasons. + */ + private async liftCallbackGuard( + conn: Connection, + receiver: Receiver + ): Promise { + if (!(await hasControl(conn, EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN))) { + return false; + } + const origin = new URL(receiver.url).origin; + const answer = await askControl( + conn, + EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN, + { origin } + ); + return answer !== undefined; + } + /** The server refused a non-routable callback, which is the rule. */ private ssrfRefusedChecks(error: JsonRpcError): ConformanceCheck[] { return [ From 8edc0cc2f493f41921592c6bab1ea8a196ac9267 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 17:29:09 +0000 Subject: [PATCH 23/27] feat(runner): say how much of a red run was never checked A missing prerequisite reports as a failure on purpose: SKIPPED is excluded from counts, exit codes and the expected-failures baseline, so a skipped row is one nobody can burn down. The cost is that a run reads as broken when it is mostly unchecked. events-webhook-delivery against a hardened server printed Passed: 2/29, 27 failed, 0 warnings where nothing was violated and 27 rows had no reachable callback to grade against. A reader cannot tell that from a server with 27 real defects. The distinction was already in the data. notTestable() writes a stable prefix and untestableCheck() sets details.untestable, both so consumers can separate "violated" from "could not be verified" without a new status. The runners just did not print it: Passed: 21/21, 0 failed, 8 warnings (8 of those unverified, not violated) The clause appears only when there is something to say, so a clean run prints exactly what it printed before. The four numbers were computed by three copies of the same four filters in server.ts, client.ts and authorization-server.ts, so a fifth number had to go in three places or in none. They now share tallyChecks/formatTally. No verdict changes: unverified counts rows already inside failed and warnings, and every exit code still keys off those, so a run that failed before fails now. Verified against a Go server, per AGENTS.md on shared infrastructure: mcpkit's kitchen-sink at eec13596 across all five events scenarios, 101 of 122 rows passing and 0 real failures, plus 10 unit cases covering the subset rule, the prefix-only path and that SKIPPED counts as neither. --- src/runner/authorization-server.ts | 13 ++-- src/runner/client.ts | 13 ++-- src/runner/server.ts | 13 ++-- src/runner/summary.test.ts | 97 ++++++++++++++++++++++++++++++ src/runner/summary.ts | 75 +++++++++++++++++++++++ 5 files changed, 184 insertions(+), 27 deletions(-) create mode 100644 src/runner/summary.test.ts create mode 100644 src/runner/summary.ts diff --git a/src/runner/authorization-server.ts b/src/runner/authorization-server.ts index 15d0ad58..6e9a4979 100644 --- a/src/runner/authorization-server.ts +++ b/src/runner/authorization-server.ts @@ -1,4 +1,5 @@ import { promises as fs } from 'fs'; +import { formatTally, tallyChecks } from './summary'; import path from 'path'; import { ConformanceCheck, SpecVersion } from '../types'; import { @@ -70,12 +71,8 @@ export function printAuthorizationServerResults( denominator: number; warnings: number; } { - const denominator = checks.filter( - (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' - ).length; - const passed = checks.filter((c) => c.status === 'SUCCESS').length; - const failed = checks.filter((c) => c.status === 'FAILURE').length; - const warnings = checks.filter((c) => c.status === 'WARNING').length; + const tally = tallyChecks(checks); + const { passed, failed, warnings, denominator } = tally; if (verbose) { console.log(JSON.stringify(checks, null, 2)); @@ -84,9 +81,7 @@ export function printAuthorizationServerResults( } console.log(`\nTest Results:`); - console.log( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` - ); + console.log(formatTally(tally)); if (failed > 0) { console.log('\n=== Failed Checks ==='); diff --git a/src/runner/client.ts b/src/runner/client.ts index af10657f..161441e4 100644 --- a/src/runner/client.ts +++ b/src/runner/client.ts @@ -1,4 +1,5 @@ import { spawn } from 'child_process'; +import { formatTally, tallyChecks } from './summary'; import { promises as fs } from 'fs'; import path from 'path'; import { @@ -264,12 +265,8 @@ export function printClientResults( warnings: number; overallFailure: boolean; } { - const denominator = checks.filter( - (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' - ).length; - const passed = checks.filter((c) => c.status === 'SUCCESS').length; - const failed = checks.filter((c) => c.status === 'FAILURE').length; - const warnings = checks.filter((c) => c.status === 'WARNING').length; + const tally = tallyChecks(checks); + const { passed, failed, warnings, denominator } = tally; // Determine if there's an overall failure (failures, warnings, client timeout, or exit failure) const clientTimedOut = clientOutput?.timedOut ?? false; @@ -292,9 +289,7 @@ export function printClientResults( // Test results summary goes to stderr console.error(`\nTest Results:`); - console.error( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` - ); + console.error(formatTally(tally)); if (clientTimedOut) { console.error(`\n⚠️ CLIENT TIMED OUT - Test incomplete`); diff --git a/src/runner/server.ts b/src/runner/server.ts index 21e80ec4..eff46219 100644 --- a/src/runner/server.ts +++ b/src/runner/server.ts @@ -1,4 +1,5 @@ import { promises as fs } from 'fs'; +import { formatTally, tallyChecks } from './summary'; import path from 'path'; import { ConformanceCheck, @@ -178,12 +179,8 @@ export function printServerResults( denominator: number; warnings: number; } { - const denominator = checks.filter( - (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' - ).length; - const passed = checks.filter((c) => c.status === 'SUCCESS').length; - const failed = checks.filter((c) => c.status === 'FAILURE').length; - const warnings = checks.filter((c) => c.status === 'WARNING').length; + const tally = tallyChecks(checks); + const { passed, failed, warnings, denominator } = tally; if (verbose) { console.log(JSON.stringify(checks, null, 2)); @@ -192,9 +189,7 @@ export function printServerResults( } console.log(`\nTest Results:`); - console.log( - `Passed: ${passed}/${denominator}, ${failed} failed, ${warnings} warnings` - ); + console.log(formatTally(tally)); if (failed > 0) { console.log('\n=== Failed Checks ==='); diff --git a/src/runner/summary.test.ts b/src/runner/summary.test.ts new file mode 100644 index 00000000..c7b58c9f --- /dev/null +++ b/src/runner/summary.test.ts @@ -0,0 +1,97 @@ +import { describe, test, expect } from 'vitest'; +import type { ConformanceCheck } from '../types'; +import { untestableCheck, notTestable } from '../scenarios/untestable'; +import { formatTally, isUnverified, tallyChecks } from './summary'; + +const check = ( + id: string, + status: ConformanceCheck['status'], + extra: Partial = {} +): ConformanceCheck => ({ + id, + name: id, + description: id, + status, + timestamp: '2026-09-24T00:00:00.000Z', + specReferences: [], + ...extra +}); + +const ref = [{ specVersion: 'draft', section: 'x' }] as never; + +describe('tallyChecks', () => { + test('counts the same four numbers the runners counted before', () => { + const tally = tallyChecks([ + check('a', 'SUCCESS'), + check('b', 'SUCCESS'), + check('c', 'FAILURE'), + check('d', 'WARNING'), + check('e', 'SKIPPED') + ]); + expect(tally).toMatchObject({ + passed: 2, + failed: 1, + warnings: 1, + denominator: 3 + }); + }); + + test('unverified is a subset of failed and warnings, never an addition', () => { + const tally = tallyChecks([ + check('violated', 'FAILURE', { errorMessage: 'cursor was 42' }), + untestableCheck('missing', 'missing', 'd', 'no control', ref), + untestableCheck('soft', 'soft', 'd', 'no control', ref, 'WARNING') + ]); + expect(tally.failed).toBe(2); + expect(tally.warnings).toBe(1); + expect(tally.unverified).toBe(2); + // The verdict a runner keys its exit code off is unchanged by the split. + expect(tally.failed + tally.warnings).toBeGreaterThan(0); + }); + + test('a scenario that formats its own message still counts as unverified', () => { + const own = check('own', 'FAILURE', { + errorMessage: notTestable('the fixture exposes no control'), + details: {} + }); + expect(isUnverified(own)).toBe(true); + expect(tallyChecks([own]).unverified).toBe(1); + }); + + test('a skipped check is neither failed nor unverified', () => { + const skipped = check('skip', 'SKIPPED', { + errorMessage: notTestable('does not apply') + }); + expect(isUnverified(skipped)).toBe(false); + expect(tallyChecks([skipped])).toMatchObject({ + failed: 0, + unverified: 0, + denominator: 0 + }); + }); +}); + +describe('formatTally', () => { + test('prints what it always printed when nothing is unverified', () => { + const line = formatTally({ + passed: 9, + failed: 1, + warnings: 0, + denominator: 10, + unverified: 0 + }); + expect(line).toBe('Passed: 9/10, 1 failed, 0 warnings'); + }); + + test('says how much of a red run was never checked', () => { + const line = formatTally({ + passed: 2, + failed: 27, + warnings: 0, + denominator: 29, + unverified: 27 + }); + expect(line).toContain('27 failed'); + expect(line).toContain('27 of those unverified, not violated'); + }); +}); diff --git a/src/runner/summary.ts b/src/runner/summary.ts new file mode 100644 index 00000000..57591810 --- /dev/null +++ b/src/runner/summary.ts @@ -0,0 +1,75 @@ +/** + * One tally for every runner's results line. + * + * The `server`, `client` and `authorization-server` runners each counted the + * same four numbers from the same filters, so a fifth number had to be added in + * three places or in none. + * + * That fifth number is what a reader of a red run most wants: how much of it was + * a requirement being violated, and how much was a requirement that could not be + * checked at all. Those are different facts with the same status, because a + * missing prerequisite reports as a failure on purpose (issue #248) — SKIPPED is + * excluded from counts, exit codes and the expected-failures baseline, so a + * skipped row is one nobody can burn down. Splitting them in the output costs + * nothing and changes no verdict: `unverified` counts rows already inside + * `failed` and `warnings`, so exit codes stay exactly as they were. + */ + +import type { ConformanceCheck } from '../types'; +import { notTestable } from '../scenarios/untestable'; + +/** The prefix `notTestable()` writes, for checks that carry no details flag. */ +const NOT_TESTABLE_PREFIX = notTestable('').trim(); + +export interface CheckTally { + /** Checks that passed. */ + passed: number; + /** Checks with status FAILURE, including ones that could not be exercised. */ + failed: number; + /** Checks with status WARNING, including ones that could not be exercised. */ + warnings: number; + /** SUCCESS + FAILURE, the population the pass ratio is quoted against. */ + denominator: number; + /** + * Of `failed` + `warnings`, how many report a missing prerequisite rather + * than a violated requirement. A subset, never an addition. + */ + unverified: number; +} + +/** + * Whether a check reports a prerequisite it could not satisfy. + * + * Either signal counts: `untestableCheck()` sets `details.untestable`, and a + * scenario that formats its own message with `notTestable()` carries only the + * prefix. Reading both means a scenario cannot fall out of the tally by + * building its check the other way. + */ +export function isUnverified(check: ConformanceCheck): boolean { + if (check.status !== 'FAILURE' && check.status !== 'WARNING') return false; + if (check.details?.untestable === true) return true; + return (check.errorMessage ?? '').startsWith(NOT_TESTABLE_PREFIX); +} + +export function tallyChecks(checks: ConformanceCheck[]): CheckTally { + return { + passed: checks.filter((c) => c.status === 'SUCCESS').length, + failed: checks.filter((c) => c.status === 'FAILURE').length, + warnings: checks.filter((c) => c.status === 'WARNING').length, + denominator: checks.filter( + (c) => c.status === 'SUCCESS' || c.status === 'FAILURE' + ).length, + unverified: checks.filter(isUnverified).length + }; +} + +/** + * The results line. The unverified clause is appended only when there is one, + * so a run with nothing unverified prints exactly what it always printed. + */ +export function formatTally(tally: CheckTally): string { + const base = `Passed: ${tally.passed}/${tally.denominator}, ${tally.failed} failed, ${tally.warnings} warnings`; + return tally.unverified > 0 + ? `${base} (${tally.unverified} of those unverified, not violated)` + : base; +} From df035e532f32415604c801d12eaded4738e404d1 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 19:10:58 +0000 Subject: [PATCH 24/27] fix(events): grade reject-non-routable on a probe the scheme rule cannot refuse Both SSRF rows were graded from one refusal of the harness's http://127.0.0.1 receiver. The https rule refuses that URL on its own, so a server that never checks routability passed reject-non-routable anyway. mcpkit's conformance build was in exactly that state until panyam/mcpkit#1462 and passed the row throughout: -32602 delivery.url rejected: delivery.url must use https (got http) SUCCESS sep-9999-ssrf-reject-non-routable The row now has its own probe, run before anything can lift a guard: an https://127.0.0.1 callback aimed at a bare TCP listener that records whether the server connected. The scheme is valid, so only routability can refuse it, and the listener separates a refusal from a dial that failed afterwards. A server that skips the check and then fails its own verification POST answers -32015, which reads as a refusal from the subscribe alone. refused, never dialled SUCCESS dialled, either way FAILURE accepted, never dialled WARNING (may be delivery-time validation) Over a tunnel the server may be on another host, where its dial to 127.0.0.1 never reaches the listener, so there only -32602 counts as a refusal. The probe also runs in that mode, where the row used to be untestable. validate-callback-url keeps grading from the http refusal, since either rule refusing it is validation. The negative fixture gains rejectNonRoutableUrl. The existing "refusing the loopback callback passes the SSRF rows" case used a fixture that enforced https alone and asserted this row SUCCESS, which certified the bug; it now uses a fixture that refuses non-routable hosts, and still asserts SUCCESS. Four new cases cover the table above, all red on the old scenario. Refs panyam/mcpkit#1460 --- .../server/events/negative-delivery.test.ts | 80 +++++++++++- .../server/events/negative-fixture.ts | 41 ++++++ src/scenarios/server/events/receiver.ts | 35 ++++++ .../server/events/webhook-delivery.ts | 119 +++++++++++++----- 4 files changed, 244 insertions(+), 31 deletions(-) diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index b2744328..d98ed653 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -148,7 +148,10 @@ describe.concurrent('a server that delivers to loopback', () => { async () => { const checks = await deliveryChecks({ capability: { listChanged: true }, - descriptors: [descriptor({ name: 'hook.event', delivery: ['webhook'] })] + descriptors: [ + descriptor({ name: 'hook.event', delivery: ['webhook'] }) + ], + subscribe: { rejectNonRoutableUrl: true } }); expect(checks.get('sep-9999-ssrf-validate-callback-url')?.status).toBe( 'SUCCESS' @@ -199,6 +202,76 @@ describe.concurrent('a server that delivers to loopback', () => { ); }); +describe.concurrent('reject-non-routable, apart from the scheme rule', () => { + const hook = [descriptor({ name: 'hook.event', delivery: ['webhook'] })]; + + // The scheme rule refuses the http loopback receiver on its own, so that + // refusal used to pass this row for a server that never checks routability. + test( + 'a server enforcing https alone fails when it dials the https loopback probe', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook, + delivery: {} + }); + expect(checks.get('sep-9999-ssrf-validate-callback-url')?.status).toBe( + 'SUCCESS' + ); + const check = checks.get('sep-9999-ssrf-reject-non-routable'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('connected to it'); + }, + TIMEOUT + ); + + test( + 'an error returned after dialling the probe is not a refusal', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook, + delivery: { synchronousVerification: true } + }); + const check = checks.get('sep-9999-ssrf-reject-non-routable'); + expect(check?.status).toBe('FAILURE'); + expect(check?.errorMessage).toContain('only after connecting'); + expect(check?.details?.code).toBe(-32015); + }, + TIMEOUT + ); + + test( + 'accepting the probe without ever dialling it warns', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook + }); + const check = checks.get('sep-9999-ssrf-reject-non-routable'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('delivery-time validation'); + }, + TIMEOUT + ); + + test( + 'refusing the probe without dialling it passes', + async () => { + const checks = await deliveryChecks({ + capability: { listChanged: true }, + descriptors: hook, + subscribe: { rejectNonRoutableUrl: true }, + delivery: {} + }); + const check = checks.get('sep-9999-ssrf-reject-non-routable'); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.connections).toBe(0); + }, + TIMEOUT + ); +}); + describe.concurrent('the verification handshake', () => { // The divergence this row exists for: kitchen-sink delivers with no handshake // at all, because the handshake does not exist yet. @@ -671,7 +744,10 @@ describe.concurrent( descriptors: [ descriptor({ name: 'hook.event', delivery: ['webhook'] }) ], - subscribe: { allowCallbackOriginControl: true }, + subscribe: { + allowCallbackOriginControl: true, + rejectNonRoutableUrl: true + }, delivery: {} }); diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index 5c072814..c7169081 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -142,6 +142,13 @@ export interface SubscribeBehaviour { acceptShortSecret?: boolean; /** Accept an `http://` callback URL. */ acceptHttpUrl?: boolean; + /** + * Refuse a callback whose host is a loopback, private, link-local or + * unspecified literal, with the rejection code. Off by default so the + * default fixture enforces the scheme rule alone, which is the state that + * used to pass `reject-non-routable` on the scheme refusal. + */ + rejectNonRoutableUrl?: boolean; /** Code for a rejected secret or URL, where the document says -32602. */ rejectionCode?: number; /** Subscribe to a type whose `delivery` does not list `webhook`. */ @@ -611,6 +618,18 @@ export async function startEventsFixture( fail(rejectionCode, 'InvalidParams: `delivery.url` must be https'); return; } + if ( + !permittedCallback && + behaviour.rejectNonRoutableUrl && + typeof url === 'string' && + nonRoutableHost(url) + ) { + fail( + rejectionCode, + 'InvalidParams: `delivery.url` is not globally routable' + ); + return; + } const served = descriptors.find( (d) => (d as { name?: unknown }).name === name @@ -1187,3 +1206,25 @@ export async function readJsonBody( unknown >; } + +/** Literal hosts a hardened server refuses without resolving anything. */ +function nonRoutableHost(url: string): boolean { + let host: string; + try { + host = new URL(url).hostname.replace(/^\[|\]$/g, '').toLowerCase(); + } catch { + return false; + } + if (host === 'localhost' || host === '::1' || host === '::') return true; + const v4 = host.match(/^(\d+)\.(\d+)\.\d+\.\d+$/); + if (!v4) return /^f[cd]|^fe80:/.test(host); + const [a, b] = [Number(v4[1]), Number(v4[2])]; + return ( + a === 127 || + a === 10 || + a === 0 || + (a === 169 && b === 254) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168) + ); +} diff --git a/src/scenarios/server/events/receiver.ts b/src/scenarios/server/events/receiver.ts index 1d034d56..740be405 100644 --- a/src/scenarios/server/events/receiver.ts +++ b/src/scenarios/server/events/receiver.ts @@ -28,6 +28,7 @@ */ import http from 'node:http'; +import net from 'node:net'; import type { AddressInfo } from 'node:net'; export interface ReceivedDelivery { @@ -235,3 +236,37 @@ export async function startReceiver(host = '127.0.0.1'): Promise { } }; } + +/** + * A bare TCP listener that records whether anything connected to it, and + * nothing else. + * + * The SSRF probe points an `https://127.0.0.1` callback here. The scheme is + * valid, so the only rule that can refuse it is routability, and a refusal is + * only evidence of that rule if the server never dialled: a server that skips + * the check and then fails its own verification POST answers with an error too, + * which reads as a refusal from the subscribe alone. The listener speaks no + * TLS and answers nothing, so every connection is dropped as soon as it lands. + */ +export interface Canary { + readonly port: number; + connections(): number; + close(): Promise; +} + +export async function startCanary(): Promise { + let count = 0; + const server = net.createServer((socket) => { + count++; + socket.destroy(); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address() as AddressInfo; + return { + port, + connections: () => count, + async close() { + await new Promise((resolve) => server.close(() => resolve())); + } + }; +} diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index 0d3eaef6..20a2d6c7 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -54,6 +54,7 @@ import { import { RECEIVER_WELL_KNOWN_PATH, WRONG_CHALLENGE_ECHO, + startCanary, startReceiver, type ReceivedDelivery, type Receiver @@ -70,6 +71,13 @@ const DELIVERY_WAIT_MS = Number(process.env.EVENTS_DELIVERY_WAIT_MS ?? 20000); */ const SETTLE_MS = Number(process.env.EVENTS_DELIVERY_SETTLE_MS ?? 5000); +/** + * How long an accepted routability probe is watched for a connection. A server + * that verifies inside events/subscribe has already dialled by the time it + * answers; this covers one that verifies or delivers just after. + */ +const CANARY_WAIT_MS = Math.min(DELIVERY_WAIT_MS, 3000); + /** A public base URL forwarding to this harness, when one exists. */ const PUBLIC_BASE = process.env.EVENTS_WEBHOOK_CALLBACK_BASE; @@ -230,7 +238,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { **Needs a reachable callback**: set \`EVENTS_WEBHOOK_CALLBACK_BASE\` to a public https base URL that forwards to this harness. -**Without one, the loopback receiver is the SSRF probe.** A server that refuses \`http://127.0.0.1\` passes the SSRF rows and reports the delivery rows untestable; a server that delivers there fails the SSRF rows and supplies real deliveries for everything else. Neither outcome is a false green.`; +**Without one, the loopback receiver is the SSRF probe.** A server that refuses \`http://127.0.0.1\` passes \`validate-callback-url\` and reports the delivery rows untestable; a server that delivers there fails it and supplies real deliveries for everything else. \`reject-non-routable\` is graded separately in either mode, from an \`https://127.0.0.1\` callback aimed at a listener that records whether the server connected, since the scheme rule alone refuses the http probe.`; async run(ctx: RunContext): Promise { const conn = await ctx.connect(); @@ -338,6 +346,10 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } }; + // Routability first, on its own probe, before anything below can lift a + // guard. dedupe keeps the first row per id, so this one is authoritative. + checks.push(await this.nonRoutableCheck(subscribe, release)); + let subscribed = await subscribe(url); // --- The SSRF rows, which a loopback callback answers directly --------- @@ -424,10 +436,7 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } else { checks.push( ...untestableAll( - [ - 'sep-9999-ssrf-validate-callback-url', - 'sep-9999-ssrf-reject-non-routable' - ], + ['sep-9999-ssrf-validate-callback-url'], 'The configured callback is routable, so the refusal path was not exercised. Run without EVENTS_WEBHOOK_CALLBACK_BASE to probe it with a loopback URL.', 'WARNING' ) @@ -538,7 +547,12 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { return answer !== undefined; } - /** The server refused a non-routable callback, which is the rule. */ + /** + * The server refused the loopback http callback. Either the scheme rule or + * routability can do that, so it answers "validates callback URLs" and + * nothing narrower; nonRoutableCheck grades routability on a URL the scheme + * rule cannot refuse. + */ private ssrfRefusedChecks(error: JsonRpcError): ConformanceCheck[] { return [ eventsCheck( @@ -546,17 +560,80 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { 'The server MUST validate callback URLs.', 'SUCCESS', { details: { code: error.code, message: error.message } } - ), - eventsCheck( - 'sep-9999-ssrf-reject-non-routable', - 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA Special-Purpose Address Registries.', - 'SUCCESS', - { details: { code: error.code, message: error.message } } ) ]; } - /** The server accepted a loopback callback. Did it also deliver there? */ + /** + * `reject-non-routable`, graded from an `https://127.0.0.1` callback aimed at + * a canary listener. The https scheme takes the scheme rule out of the + * question, and the canary says whether the server dialled, which is what + * separates "refused because the address is not routable" from "tried it and + * the handshake failed". + */ + private async nonRoutableCheck( + subscribe: ( + callbackUrl: string + ) => Promise<{ id?: unknown } | { error: JsonRpcError }>, + release: (callbackUrl: string) => Promise + ): Promise { + const id = 'sep-9999-ssrf-reject-non-routable'; + const description = + 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA Special-Purpose Address Registries.'; + const canary = await startCanary(); + const probe = `https://127.0.0.1:${canary.port}/ssrf-probe-${Date.now()}`; + try { + const answered = await subscribe(probe); + const refused = 'error' in answered ? answered.error : undefined; + if (!refused) { + const deadline = Date.now() + CANARY_WAIT_MS; + while (canary.connections() === 0 && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, 100)); + } + await release(probe); + } + const dialled = canary.connections(); + + if (dialled > 0) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: refused + ? `The server answered ${refused.code} ${refused.message} for ${probe}, but only after connecting to it (${dialled} connection(s)). A callback on 127.0.0.0/8 was dialled rather than refused, so the error came from the connection failing, not from the address check.` + : `The server accepted ${probe} and connected to it (${dialled} connection(s)). 127.0.0.0/8 is not globally routable, so a caller can aim requests at services the server can reach and the caller cannot.`, + details: { url: probe, connections: dialled, code: refused?.code } + }); + } + if (!refused) { + return eventsCheck(id, description, 'WARNING', { + errorMessage: `${probe} was accepted at subscribe time and nothing connected to it within ${CANARY_WAIT_MS}ms. That may be delivery-time validation rather than a missing check, and an idle event type looks the same.`, + details: { url: probe } + }); + } + // Over a tunnel the server may not share this host, so its dial to + // 127.0.0.1 would never reach the canary. Only a parameter error is + // unambiguous then. + if (PUBLIC_BASE && refused.code !== -32602) { + return eventsCheck(id, description, 'WARNING', { + errorMessage: `${probe} was refused with ${refused.code} ${refused.message}. With EVENTS_WEBHOOK_CALLBACK_BASE set the server may be on another host, where a failed connection and an address refusal look the same from here; only -32602 settles it.`, + details: { url: probe, code: refused.code, message: refused.message } + }); + } + return eventsCheck(id, description, 'SUCCESS', { + details: { + url: probe, + code: refused.code, + message: refused.message, + connections: 0 + } + }); + } finally { + await canary.close(); + } + } + + /** + * The server accepted a loopback callback. Did it also deliver there? Only + * `validate-callback-url` is graded here; routability has its own probe. + */ private ssrfDeliveredChecks( delivered: boolean, url: string @@ -570,14 +647,6 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { { errorMessage: `The subscribe for ${url} was accepted, but nothing was delivered, so the harness cannot tell delivery-time hardening from an idle event type.` } - ), - eventsCheck( - 'sep-9999-ssrf-reject-non-routable', - 'Servers SHOULD reject URLs whose resolved IP is not globally routable.', - 'WARNING', - { - errorMessage: `A loopback \`delivery.url\` was accepted at subscribe time. Nothing was delivered to it, so this may be delivery-time validation rather than a missing check.` - } ) ]; } @@ -589,14 +658,6 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { { errorMessage: `The server POSTed to ${url}, a loopback address. A callback URL pointing inside the server's own network was neither refused at subscribe time nor at delivery time.` } - ), - eventsCheck( - 'sep-9999-ssrf-reject-non-routable', - 'Servers SHOULD reject URLs whose resolved IP is not globally routable per the IANA Special-Purpose Address Registries.', - 'FAILURE', - { - errorMessage: `Delivered to ${url}. 127.0.0.0/8 is not globally routable, so this is the SSRF case the rule exists to stop: a caller can aim deliveries at services the server can reach and the caller cannot.` - } ) ]; } From 863b20bab3321ca6940882cd77546603269a0751 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 19:13:50 +0000 Subject: [PATCH 25/27] feat(events): probe the -32013 row on the type a server says is capped error-resource-exhausted was graded only inside the concurrency probe, which opens three streams on one event type and needs all three to stay open for stream-exempt-from-concurrency-cap. A server capped on that type passes the error row and fails the MUST beside it, so on one type the two cannot both pass, and a server that caps anything else never has the row provoked at all. mcpkit reports it untestable for exactly that reason (panyam/mcpkit#1461), though it answers -32013 with data.limit on every subscribe path. New read-only control, events_conformance_quota, answering {"name": "", "max": }. When present, the scenario opens streams on that type one at a time, up to max+1, and grades the first refusal: -32013 with data.limit SUCCESS -32013 without it WARNING another code FAILURE max+1 opened, no refusal untestable, naming the cap that never bit A refusal before max+1 counts, since other scenarios may hold subscriptions under the same principal. The probe's row is pushed ahead of the concurrency probe's, so dedupe keeps it; without the control the concurrency probe's opportunistic grading stands as before, and its untestable message now names the control. The negative fixture gains a per-type quota with the control, and four cases cover the table and the absent control, all red on the old scenario. The existing maxConcurrent cases are unchanged. Refs panyam/mcpkit#1461 --- src/scenarios/server/events/helpers.ts | 12 ++ .../server/events/negative-fixture.ts | 57 +++++++++- .../server/events/negative-push.test.ts | 58 ++++++++++ src/scenarios/server/events/push.ts | 107 +++++++++++++++++- 4 files changed, 229 insertions(+), 5 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 87bcda45..5cd20e76 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -233,6 +233,18 @@ export const EVENTS_CONTROL_SUBSCRIBE_AS = 'events_conformance_subscribe_as'; export const EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN = 'events_conformance_allow_callback_origin'; +/** + * Reports one per-event-type subscription cap the server enforces, as JSON text + * `{"name": "", "max": }`. Read-only. + * + * `-32013` is only reachable by exceeding a limit, and nothing in the protocol + * says where a server's limits are. The concurrency probe cannot find one for + * the suite: it opens three streams on one type and needs all three to stay + * open, so a server capped there fails the MUST beside it. Knowing the capped + * type lets the quota be probed on its own. + */ +export const EVENTS_CONTROL_QUOTA = 'events_conformance_quota'; + /** Reports whether a given principal's subscription is still registered. */ export const EVENTS_CONTROL_SUBSCRIPTION_EXISTS = 'events_conformance_subscription_exists'; diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index c7169081..730e833d 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -284,9 +284,28 @@ export interface DurabilityBehaviour { restartNoop?: boolean; } +/** + * A per-event-type subscription cap on streams, the shape kitchen-sink's + * `Quota` has, reported through `events_conformance_quota`. + */ +export interface QuotaBehaviour { + /** Event type the cap applies to. */ + name: string; + /** Streams allowed at once on that type. */ + max: number; + /** Refuse past `max`. Off stands for a server that reports a cap it does not enforce. */ + enforce?: boolean; + /** `data.limit` on the refusal; null omits it. Defaults to `subscriptions`. */ + limitName?: string | null; + /** Expose `events_conformance_quota`. Defaults to true. */ + control?: boolean; +} + export interface EventsFixtureOptions { /** Expose the durability controls as tools. */ durability?: DurabilityBehaviour; + /** A per-type cap on streams, and optionally the control that reports it. */ + quota?: QuotaBehaviour; /** * Raw value to declare at `capabilities.extensions["io.modelcontextprotocol/events"]`; * omit for no declaration. @@ -377,6 +396,7 @@ export async function startEventsFixture( /** Open streams, so close() can tear them down instead of hanging on them. */ const openStreams = new Set<{ res: ServerResponse; stop: () => void }>(); let liveStreams = 0; + const liveByName = new Map(); const server = createServer(async (req, res) => { if (req.method !== 'POST') { @@ -421,12 +441,28 @@ export async function startEventsFixture( inputSchema: obj }); } + if (opts.quota && opts.quota.control !== false) { + tools.push({ name: 'events_conformance_quota', inputSchema: obj }); + } if (tools.length > 0 || opts.durability) { send({ tools }); return; } } + if ( + method === 'tools/call' && + params.name === 'events_conformance_quota' && + opts.quota && + opts.quota.control !== false + ) { + const text = JSON.stringify({ + name: opts.quota.name, + max: opts.quota.max + }); + send({ content: [{ type: 'text', text }] }); + return; + } if ( method === 'tools/call' && params.name === 'events_conformance_allow_callback_origin' @@ -780,17 +816,36 @@ export async function startEventsFixture( ); return; } + const quota = opts.quota; + const capped = quota !== undefined && quota.name === name; + if ( + capped && + quota.enforce !== false && + (liveByName.get(quota.name) ?? 0) >= quota.max + ) { + const limit = + quota.limitName === undefined ? 'subscriptions' : quota.limitName; + fail( + -32013, + 'ResourceExhausted: subscription quota reached', + limit === null ? undefined : { limit, max: quota.max } + ); + return; + } if (behaviour.answerJson) { send({}); return; } liveStreams += 1; - const entry = openStream(res, id, String(name), behaviour); + const key = String(name); + if (capped) liveByName.set(key, (liveByName.get(key) ?? 0) + 1); + const entry = openStream(res, id, key, behaviour); openStreams.add(entry); const done = () => { if (!openStreams.delete(entry)) return; liveStreams -= 1; + if (capped) liveByName.set(key, (liveByName.get(key) ?? 1) - 1); entry.stop(); }; req.on('close', done); diff --git a/src/scenarios/server/events/negative-push.test.ts b/src/scenarios/server/events/negative-push.test.ts index b78bb64d..22c656bd 100644 --- a/src/scenarios/server/events/negative-push.test.ts +++ b/src/scenarios/server/events/negative-push.test.ts @@ -316,6 +316,64 @@ describe.concurrent('the -32013 error-table row', () => { }); }); +describe.concurrent( + 'the -32013 row, probed on the type a server says is capped', + () => { + // Two push types: the scenario's target, which the concurrency probe opens + // three streams on, and a capped one it must leave alone. + const withQuota = ( + quota: EventsFixtureOptions['quota'] + ): EventsFixtureOptions => ({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'push.event', delivery: ['push'] }), + descriptor({ name: 'capped.event', delivery: ['push'] }) + ], + quota + }); + + test('an enforced cap that names its limit passes, beside a passing concurrency row', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 2 }) + ); + const check = checks.get('sep-9999-error-resource-exhausted'); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.limit).toBe('subscriptions'); + expect( + checks.get('sep-9999-stream-exempt-from-concurrency-cap')?.status + ).toBe('SUCCESS'); + }); + + test('a refusal without data.limit warns', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 1, limitName: null }) + ); + const check = checks.get('sep-9999-error-resource-exhausted'); + expect(check?.status).toBe('WARNING'); + expect(check?.errorMessage).toContain('which quota it hit'); + }); + + test('a reported cap that is never enforced is untestable, and says so', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 2, enforce: false }) + ); + const check = checks.get('sep-9999-error-resource-exhausted'); + expect(check?.status).toBe('FAILURE'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('3 streams opened'); + }); + + test('without the control the row stays untestable and names it', async () => { + const checks = await pushChecks( + withQuota({ name: 'capped.event', max: 2, control: false }) + ); + const check = checks.get('sep-9999-error-resource-exhausted'); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('events_conformance_quota'); + }); + } +); + describe.concurrent('concurrency and cancellation', () => { // The cap kitchen-sink applies to streams, which the document exempts them // from: the first stream confirms and the other two are refused -32013. diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts index 6322ea58..a1a400d6 100644 --- a/src/scenarios/server/events/push.ts +++ b/src/scenarios/server/events/push.ts @@ -33,7 +33,7 @@ */ import { ClientScenario, ConformanceCheck } from '../../../types'; -import type { RunContext } from '../../../connection'; +import type { Connection, RunContext } from '../../../connection'; import { untestableCheck } from '../../untestable'; import { EVENTS_ACTIVE_NOTIFICATION, @@ -41,6 +41,8 @@ import { EVENTS_CONTROL_YIELD_ERROR, EVENTS_CONTROL_YIELD_GAP, EVENTS_CONTROL_TERMINATE, + EVENTS_CONTROL_QUOTA, + askControl, deliveryModes, type EventDescriptor, hasControl, @@ -198,12 +200,14 @@ export class EventsPushScenario implements ClientScenario { ); } + const quota = await this.readQuota(conn); return await this.streamChecks( ctx, name, args, controls, - listed.descriptors + listed.descriptors, + quota ); } finally { await conn.close(); @@ -215,7 +219,8 @@ export class EventsPushScenario implements ClientScenario { name: string, args: Record, controls: { error: boolean; gap: boolean; terminate: boolean }, - descriptors: EventDescriptor[] + descriptors: EventDescriptor[], + quota?: QuotaReport ): Promise { const checks: ConformanceCheck[] = []; const session = await openEventStream( @@ -349,6 +354,9 @@ export class EventsPushScenario implements ClientScenario { controls.terminate )) ); + // Ahead of the concurrency probe, whose own -32013 grading is the + // fallback for a server without the control; dedupe keeps the first. + if (quota) checks.push(await this.quotaCheck(ctx, quota, descriptors)); checks.push(...(await this.concurrencyChecks(ctx, name, args))); checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); return dedupe(checks); @@ -958,6 +966,94 @@ export class EventsPushScenario implements ClientScenario { } } + /** + * What `events_conformance_quota` reported, or undefined when the server has + * no such control, which is the normal case. + */ + private async readQuota(conn: Connection): Promise { + if (!(await hasControl(conn, EVENTS_CONTROL_QUOTA))) return undefined; + const text = await askControl(conn, EVENTS_CONTROL_QUOTA, {}); + let parsed: unknown; + try { + parsed = text === undefined ? undefined : JSON.parse(text); + } catch { + parsed = undefined; + } + if ( + isObject(parsed) && + typeof parsed.name === 'string' && + typeof parsed.max === 'number' && + Number.isInteger(parsed.max) && + parsed.max >= 0 + ) { + return { name: parsed.name, max: parsed.max }; + } + return { malformed: text ?? '(no text, or the call failed)' }; + } + + /** + * `-32013`, provoked on the type the server says is capped. + * + * Opens streams one at a time, up to one past the reported cap, and stops at + * the first refusal. Refused earlier than the cap still counts, since other + * scenarios in the run may hold subscriptions under the same principal. + */ + private async quotaCheck( + ctx: RunContext, + quota: QuotaReport, + descriptors: EventDescriptor[] + ): Promise { + const id = 'sep-9999-error-resource-exhausted'; + const description = + '`-32013 ResourceExhausted` — a server-imposed limit or quota was reached. `data.limit` names it (e.g. `"subscriptions"`).'; + const untestable = (reason: string) => + untestableCheck(id, id, description, reason, [EVENTS_SPEC_REF]); + + if ('malformed' in quota) { + return untestable( + `\`${EVENTS_CONTROL_QUOTA}\` answered ${quota.malformed}, where \`{"name": "", "max": }\` was expected, so no capped type was known to probe.` + ); + } + const target = descriptors.find((d) => descriptorName(d) === quota.name); + const args = target ? minimalArguments(target) : undefined; + if (!target || args === undefined) { + return untestable( + target + ? `\`${quota.name}\`, the type \`${EVENTS_CONTROL_QUOTA}\` reported as capped, declares required \`inputSchema\` properties the harness cannot satisfy.` + : `\`${EVENTS_CONTROL_QUOTA}\` reported \`${quota.name}\` as capped, but \`events/list\` does not serve it.` + ); + } + + const sessions: StreamSession[] = []; + try { + let refusal: StreamSession['error']; + for (let i = 0; i <= quota.max && !refusal; i++) { + const s = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name: quota.name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + sessions.push(s); + refusal = s.error; + } + if (!refusal) { + return untestable( + `\`${EVENTS_CONTROL_QUOTA}\` reported a cap of ${quota.max} on \`${quota.name}\`, and ${sessions.length} streams opened on it without a refusal, so the cap it reported was never reached.` + ); + } + if (refusal.code !== EVENTS_RESOURCE_EXHAUSTED) { + return eventsCheck(id, description, 'FAILURE', { + errorMessage: `Stream ${sessions.length} on \`${quota.name}\`, past its reported cap of ${quota.max}, was refused with ${refusal.code} ${refusal.message}. A refusal for a quota is \`${EVENTS_RESOURCE_EXHAUSTED}\`.`, + details: { code: refusal.code, message: refusal.message } + }); + } + return this.resourceExhaustedCheck([refusal]); + } finally { + await Promise.all(sessions.map((s) => s.cancel())); + } + } + /** * The `-32013` row from the error table. * @@ -978,7 +1074,7 @@ export class EventsPushScenario implements ClientScenario { id, id, description, - 'No request in this run was refused for a server-imposed limit, so the code was never provoked. A server that caps concurrent subscriptions answers it for the third stream above.', + `No request in this run was refused for a server-imposed limit, so the code was never provoked. A server that caps concurrent subscriptions answers it for the third stream above, or can name a capped event type through the \`${EVENTS_CONTROL_QUOTA}\` control so the cap is probed on its own.`, [EVENTS_SPEC_REF] ); } @@ -1048,6 +1144,9 @@ export class EventsPushScenario implements ClientScenario { } /** Keep the first check emitted per id, so a fallback path cannot double-report. */ +/** What `events_conformance_quota` answered: a capped type, or garbage. */ +type QuotaReport = { name: string; max: number } | { malformed: string }; + function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { const seen = new Set(); return checks.filter((c) => { From 45e4ede806f563fc897a3cfc90ea6244988cc1e2 Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 20:20:22 +0000 Subject: [PATCH 26/27] feat(events): ask for the gap and terminated envelopes on the scenario's own subscription envelope-gap and envelope-terminated reported untestable against every server, because nothing in a run produces either. The push scenario solved the same problem with yield_gap and terminate, but those cannot carry over: in mcpkit a source's gap and terminal signals reach push streams only, so no webhook envelope results, and terminate ends a whole event type, which for a webhook-capable type ends it for every scenario after this one. Two per-subscription controls instead, taking { id }: events_conformance_webhook_gap and events_conformance_webhook_terminate. The scenario fires them after the retry and redirect probes, terminate last since it ends the main subscription, and waits up to 5s for each envelope. envelope arrived, well formed SUCCESS arrived without cursor / error WARNING acknowledged, nothing arrived gap WARNING, terminated FAILURE control absent or declined untestable, naming the control The envelope rows now grade after the probes, from what arrived, so the signalled envelopes also count toward the discriminator, signing and webhook-id rows. The negative fixture gains both controls, reusing its envelope sender with the event and handshake switched off. Four cases, all red on the old scenario; the malformed-envelope case asserts on the graded body as well as the status, since an untestable row is also a WARNING. Refs panyam/mcpkit#1463 --- src/scenarios/server/events/helpers.ts | 16 +++ .../server/events/negative-delivery.test.ts | 83 ++++++++++++ .../server/events/negative-fixture.ts | 82 +++++++++++- .../server/events/webhook-delivery.ts | 126 ++++++++++++++---- 4 files changed, 282 insertions(+), 25 deletions(-) diff --git a/src/scenarios/server/events/helpers.ts b/src/scenarios/server/events/helpers.ts index 5cd20e76..55ccd462 100644 --- a/src/scenarios/server/events/helpers.ts +++ b/src/scenarios/server/events/helpers.ts @@ -245,6 +245,22 @@ export const EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN = */ export const EVENTS_CONTROL_QUOTA = 'events_conformance_quota'; +/** + * Send a `{type:"gap"}` envelope to one webhook subscription, `{ id }`, + * answering the cursor it carried. Per subscription rather than per event type + * because a source's gap signal reaches push streams, and a webhook subscriber + * hears of one only when the server posts to it. + */ +export const EVENTS_CONTROL_WEBHOOK_GAP = 'events_conformance_webhook_gap'; + +/** + * End one webhook subscription, `{ id }`, sending it `{type:"terminated"}`. + * Per subscription so the scenario ends only its own, where terminating an + * event type would end it for every scenario after. + */ +export const EVENTS_CONTROL_WEBHOOK_TERMINATE = + 'events_conformance_webhook_terminate'; + /** Reports whether a given principal's subscription is still registered. */ export const EVENTS_CONTROL_SUBSCRIPTION_EXISTS = 'events_conformance_subscription_exists'; diff --git a/src/scenarios/server/events/negative-delivery.test.ts b/src/scenarios/server/events/negative-delivery.test.ts index d98ed653..97f7d528 100644 --- a/src/scenarios/server/events/negative-delivery.test.ts +++ b/src/scenarios/server/events/negative-delivery.test.ts @@ -596,6 +596,89 @@ describe.concurrent('control envelopes', () => { ); }); +describe.concurrent( + 'envelopes a control asks for, on this subscription', + () => { + const withControls = ( + controls: EventsFixtureOptions['webhookEnvelopeControls'] + ): EventsFixtureOptions => ({ + ...delivering(), + webhookEnvelopeControls: controls + }); + + test( + 'a gap and a termination on request both grade, and count as signed envelopes', + async () => { + const checks = await deliveryChecks( + withControls({ gap: true, terminate: true }) + ); + expect(checks.get('sep-9999-envelope-gap')?.status).toBe('SUCCESS'); + expect(checks.get('sep-9999-envelope-terminated')?.status).toBe( + 'SUCCESS' + ); + expect( + checks.get('sep-9999-envelope-signed-like-deliveries')?.status + ).toBe('SUCCESS'); + }, + TIMEOUT + ); + + test( + 'envelopes missing their cursor or error warn', + async () => { + const checks = await deliveryChecks( + withControls({ gap: { cursor: null }, terminate: { error: null } }) + ); + // Graded from the envelope that arrived, not reported untestable, which + // is also a WARNING and would pass the status check on its own. + const gap = checks.get('sep-9999-envelope-gap'); + expect(gap?.status).toBe('WARNING'); + expect(gap?.details?.body).toMatchObject({ type: 'gap', cursor: null }); + const terminated = checks.get('sep-9999-envelope-terminated'); + expect(terminated?.status).toBe('WARNING'); + expect(terminated?.details?.body).toMatchObject({ + type: 'terminated', + error: null + }); + }, + TIMEOUT + ); + + test( + 'a control that acknowledges and sends nothing is graded, not untestable', + async () => { + const checks = await deliveryChecks( + withControls({ gap: 'silent', terminate: 'silent' }) + ); + const gap = checks.get('sep-9999-envelope-gap'); + expect(gap?.status).toBe('WARNING'); + expect(gap?.details?.untestable).toBeUndefined(); + const terminated = checks.get('sep-9999-envelope-terminated'); + expect(terminated?.status).toBe('FAILURE'); + expect(terminated?.details?.untestable).toBeUndefined(); + expect(terminated?.errorMessage).toContain('never told'); + }, + TIMEOUT + ); + + test( + 'without the controls both rows stay untestable and name them', + async () => { + const checks = await deliveryChecks(delivering()); + const gap = checks.get('sep-9999-envelope-gap'); + expect(gap?.details?.untestable).toBe(true); + expect(gap?.errorMessage).toContain('events_conformance_webhook_gap'); + const terminated = checks.get('sep-9999-envelope-terminated'); + expect(terminated?.details?.untestable).toBe(true); + expect(terminated?.errorMessage).toContain( + 'events_conformance_webhook_terminate' + ); + }, + TIMEOUT + ); + } +); + describe.concurrent('confirming intent without a handshake', () => { // The document allows four consent paths and the harness can only see two of // them. Before this, a server that read the receiver's well-known document and diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index 730e833d..afbd87e3 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -301,9 +301,21 @@ export interface QuotaBehaviour { control?: boolean; } +/** + * `events_conformance_webhook_gap` / `_terminate`, which signal one webhook + * subscription by id. `true` sends the conformant envelope, an object + * overrides its fields, and `'silent'` acknowledges and sends nothing. + */ +export interface WebhookEnvelopeControls { + gap?: true | { cursor?: unknown } | 'silent'; + terminate?: true | { error?: unknown } | 'silent'; +} + export interface EventsFixtureOptions { /** Expose the durability controls as tools. */ durability?: DurabilityBehaviour; + /** Expose the per-subscription webhook envelope controls. */ + webhookEnvelopeControls?: WebhookEnvelopeControls; /** A per-type cap on streams, and optionally the control that reports it. */ quota?: QuotaBehaviour; /** @@ -377,7 +389,14 @@ export async function startEventsFixture( /** Live subscriptions, keyed the way the document keys them. */ const subscriptions = new Map< string, - { id: string; noExpiry?: boolean; at?: number } + { + id: string; + noExpiry?: boolean; + at?: number; + url?: string; + secret?: unknown; + name?: string; + } >(); let generation = 1; /** Deliveries still in flight, so close() can settle rather than abandon. */ @@ -444,12 +463,66 @@ export async function startEventsFixture( if (opts.quota && opts.quota.control !== false) { tools.push({ name: 'events_conformance_quota', inputSchema: obj }); } + if (opts.webhookEnvelopeControls?.gap) { + tools.push({ + name: 'events_conformance_webhook_gap', + inputSchema: obj + }); + } + if (opts.webhookEnvelopeControls?.terminate) { + tools.push({ + name: 'events_conformance_webhook_terminate', + inputSchema: obj + }); + } if (tools.length > 0 || opts.durability) { send({ tools }); return; } } + if ( + method === 'tools/call' && + opts.webhookEnvelopeControls && + (params.name === 'events_conformance_webhook_gap' || + params.name === 'events_conformance_webhook_terminate') + ) { + const isGap = params.name === 'events_conformance_webhook_gap'; + const mode = isGap + ? opts.webhookEnvelopeControls.gap + : opts.webhookEnvelopeControls.terminate; + const toolArgs = (params.arguments ?? {}) as Record; + const entry = [...subscriptions.entries()].find( + ([, sub]) => sub.id === toolArgs.id + ); + const sub = entry?.[1]; + if (!mode || !entry || !sub?.url) { + send({ + content: [{ type: 'text', text: 'no such webhook subscription' }], + isError: true + }); + return; + } + if (!isGap) subscriptions.delete(entry[0]); + if (mode !== 'silent') { + const run = deliverToCallback( + sub.url, + sub.secret, + sub.id, + sub.name ?? '', + { + verify: false, + sendEvent: false, + ...(isGap ? { gapEnvelope: mode } : { terminatedEnvelope: mode }) + } as DeliveryBehaviour + ).finally(() => inFlight.delete(run)); + inFlight.add(run); + } + send({ + content: [{ type: 'text', text: isGap ? 'cursor_after_gap' : 'ok' }] + }); + return; + } if ( method === 'tools/call' && params.name === 'events_conformance_quota' && @@ -718,7 +791,12 @@ export async function startEventsFixture( : behaviour.nonIdempotentId ? `sub_${++mintedIds}_${hashKey(effectiveKey)}` : hashKey(effectiveKey); - subscriptions.set(effectiveKey, { id }); + subscriptions.set(effectiveKey, { + id, + url: typeof url === 'string' ? url : undefined, + secret: delivery.secret, + name: String(name) + }); // Clamp rather than reject, and never hand back no-expiry unasked. const cap = 7 * 24 * 3600_000; diff --git a/src/scenarios/server/events/webhook-delivery.ts b/src/scenarios/server/events/webhook-delivery.ts index 20a2d6c7..8632641c 100644 --- a/src/scenarios/server/events/webhook-delivery.ts +++ b/src/scenarios/server/events/webhook-delivery.ts @@ -44,6 +44,8 @@ import { eventsCheck, eventsListAll, EVENTS_CONTROL_ALLOW_CALLBACK_ORIGIN, + EVENTS_CONTROL_WEBHOOK_GAP, + EVENTS_CONTROL_WEBHOOK_TERMINATE, askControl, hasControl, firstSupporting, @@ -78,6 +80,9 @@ const SETTLE_MS = Number(process.env.EVENTS_DELIVERY_SETTLE_MS ?? 5000); */ const CANARY_WAIT_MS = Math.min(DELIVERY_WAIT_MS, 3000); +/** How long to wait for a gap or terminated envelope a control asked for. */ +const ENVELOPE_WAIT_MS = Math.min(DELIVERY_WAIT_MS, 5000); + /** A public base URL forwarding to this harness, when one exists. */ const PUBLIC_BASE = process.env.EVENTS_WEBHOOK_CALLBACK_BASE; @@ -513,9 +518,19 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { checks.push(this.callbackEndpointErrorCheck(endpointFailure)); checks.push(...this.transportChecks(all, subscriptionId)); checks.push(...this.signatureChecks(all, secret.bytes)); - checks.push(...this.envelopeChecks(all)); checks.push(...(await this.redirectChecks(receiver, subscribe, release))); checks.push(...(await this.retryChecks(receiver, subscribe, release))); + // Last, because terminating ends the subscription everything above + // delivered to. The envelope rows grade from what arrived after, so the + // signalled gap and terminated envelopes count toward the discriminator, + // signing and id rows too. + const signalled = await this.signalEnvelopes( + conn, + receiver, + path, + subscriptionId + ); + checks.push(...this.envelopeChecks(receiver.on(path), signalled)); return dedupe(checks); } finally { await release(url); @@ -1092,7 +1107,44 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { } /** Control envelopes versus event bodies. */ - private envelopeChecks(all: ReceivedDelivery[]): ConformanceCheck[] { + /** + * Ask the server to send this subscription a gap, then to end it, and wait + * for each envelope. Absent controls are the normal case; the rows then fall + * back to whatever the server sent unasked. + */ + private async signalEnvelopes( + conn: Connection, + receiver: Receiver, + path: string, + subscriptionId: unknown + ): Promise { + const signal = async ( + tool: string, + type: string + ): Promise => { + if (!(await hasControl(conn, tool))) return 'absent'; + if (typeof subscriptionId !== 'string') return 'refused'; + if ((await askControl(conn, tool, { id: subscriptionId })) === undefined) + return 'refused'; + await receiver.waitFor( + path, + (d) => d.json?.type === type, + ENVELOPE_WAIT_MS + ); + return 'sent'; + }; + const gap = await signal(EVENTS_CONTROL_WEBHOOK_GAP, 'gap'); + const terminate = await signal( + EVENTS_CONTROL_WEBHOOK_TERMINATE, + 'terminated' + ); + return { gap, terminate }; + } + + private envelopeChecks( + all: ReceivedDelivery[], + signalled: SignalledEnvelopes + ): ConformanceCheck[] { const out: ConformanceCheck[] = []; const envelopes = all.filter((d) => typeof d.json?.type === 'string'); const events = all.filter( @@ -1183,23 +1235,31 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ); } - // A gap and a termination cannot be provoked from the client side, so these - // usually report untestable. They are still graded when a server sends one - // unasked, because the envelope is right here in what arrived. + // A gap and a termination cannot be provoked from the client side. The + // webhook controls provoke one each on this subscription; without them the + // rows are still graded when a server sends one unasked. + const gapDesc = + 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes. The client persists `cursor` and treats it as `truncated: true`.'; const gap = all.find((d) => d.json?.type === 'gap'); out.push( !gap - ? untestableCheck( - 'sep-9999-envelope-gap', - 'sep-9999-envelope-gap', - 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes.', - 'No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture that can expire its replay window on demand.', - [EVENTS_SPEC_REF], - 'WARNING' - ) + ? signalled.gap === 'sent' + ? eventsCheck('sep-9999-envelope-gap', gapDesc, 'WARNING', { + errorMessage: `\`${EVENTS_CONTROL_WEBHOOK_GAP}\` acknowledged, and no \`gap\` envelope arrived within ${ENVELOPE_WAIT_MS}ms.` + }) + : untestableCheck( + 'sep-9999-envelope-gap', + 'sep-9999-envelope-gap', + gapDesc, + signalled.gap === 'refused' + ? `No retention gap occurred during the run, and \`${EVENTS_CONTROL_WEBHOOK_GAP}\` declined to signal one.` + : `No retention gap occurred during the run, and the harness cannot force one from the client side. Needs a fixture exposing the \`${EVENTS_CONTROL_WEBHOOK_GAP}\` control.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) : eventsCheck( 'sep-9999-envelope-gap', - 'A `gap` envelope `{"type":"gap","cursor":""}` is sent when a gap is detected between refreshes. The client persists `cursor` and treats it as `truncated: true`.', + gapDesc, typeof gap.json?.cursor === 'string' ? 'SUCCESS' : 'WARNING', { errorMessage: @@ -1211,20 +1271,33 @@ export class EventsWebhookDeliveryScenario implements ClientScenario { ) ); + const terminatedDesc = + 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended (e.g., authorization revoked). The subscription no longer exists server-side.'; const terminated = all.find((d) => d.json?.type === 'terminated'); out.push( !terminated - ? untestableCheck( - 'sep-9999-envelope-terminated', - 'sep-9999-envelope-terminated', - 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended.', - 'The subscription was not terminated during the run. Needs a server that can revoke authorization or remove an event type mid-run.', - [EVENTS_SPEC_REF], - 'WARNING' - ) + ? signalled.terminate === 'sent' + ? eventsCheck( + 'sep-9999-envelope-terminated', + terminatedDesc, + 'FAILURE', + { + errorMessage: `\`${EVENTS_CONTROL_WEBHOOK_TERMINATE}\` acknowledged ending the subscription, and no \`terminated\` envelope arrived within ${ENVELOPE_WAIT_MS}ms, so the receiver was never told.` + } + ) + : untestableCheck( + 'sep-9999-envelope-terminated', + 'sep-9999-envelope-terminated', + terminatedDesc, + signalled.terminate === 'refused' + ? `The subscription was not terminated during the run, and \`${EVENTS_CONTROL_WEBHOOK_TERMINATE}\` declined to end it.` + : `The subscription was not terminated during the run. Needs a server that can revoke authorization or remove an event type mid-run, or a fixture exposing the \`${EVENTS_CONTROL_WEBHOOK_TERMINATE}\` control.`, + [EVENTS_SPEC_REF], + 'WARNING' + ) : eventsCheck( 'sep-9999-envelope-terminated', - 'A `terminated` envelope `{"type":"terminated","error":{...}}` is sent when the subscription has ended (e.g., authorization revoked). The subscription no longer exists server-side.', + terminatedDesc, isObject(terminated.json?.error) ? 'SUCCESS' : 'WARNING', { errorMessage: isObject(terminated.json?.error) @@ -1596,6 +1669,13 @@ function isVerificationEnvelope(d: ReceivedDelivery): boolean { } /** Keep the first check emitted per id, so a fallback path cannot double-report. */ +/** + * What became of asking for each control envelope: no control, a control that + * declined, or a control that acknowledged (whether or not anything arrived). + */ +type SignalOutcome = 'absent' | 'refused' | 'sent'; +type SignalledEnvelopes = { gap: SignalOutcome; terminate: SignalOutcome }; + function dedupe(checks: ConformanceCheck[]): ConformanceCheck[] { const seen = new Set(); return checks.filter((c) => { From 2323122b9198009eee7bc83899c388586836928a Mon Sep 17 00:00:00 2001 From: Sri Panyam Date: Thu, 24 Sep 2026 21:53:36 +0000 Subject: [PATCH 27/27] feat(events): grade truncated-false-when-no-replay on push Only the poll scenario graded this row, and only against its own target, so a server whose replay-capable types it polls reported it SKIPPED while its push streams broke the rule. mcpkit did exactly that (panyam/mcpkit#1468): a gap on a cursorless type sent notifications/events/active {cursor: null, truncated: true}. The push scenario now finds a type without replay by opening each push-capable type until one confirms its stream with cursor: null, the push analogue of poll reading cursor: null off its result, since events/list declares nothing about replay. It asks for a gap there with yield_gap and watches 1.5s: no truncated:true active SUCCESS truncated:true active WARNING (SHOULD), with the frame every type replays SKIPPED, as on poll no yield_gap control untestable, naming it run() fills the row in on every early return, so runs stay comparable. The negative fixture gains one no-replay type and the yield_gap control, which sends the conformant fresh active for any other type. Five cases, all red on the old scenario. Refs panyam/mcpkit#1468 --- .../server/events/negative-fixture.ts | 66 ++++++++- .../server/events/negative-push.test.ts | 54 ++++++++ src/scenarios/server/events/push.ts | 127 +++++++++++++++++- 3 files changed, 240 insertions(+), 7 deletions(-) diff --git a/src/scenarios/server/events/negative-fixture.ts b/src/scenarios/server/events/negative-fixture.ts index afbd87e3..f85998ba 100644 --- a/src/scenarios/server/events/negative-fixture.ts +++ b/src/scenarios/server/events/negative-fixture.ts @@ -316,6 +316,14 @@ export interface EventsFixtureOptions { durability?: DurabilityBehaviour; /** Expose the per-subscription webhook envelope controls. */ webhookEnvelopeControls?: WebhookEnvelopeControls; + /** + * One event type without replay: its streams confirm with `cursor: null`, + * and `events_conformance_yield_gap` is exposed. For that type the control + * sends `active{cursor: null, truncated: true}` when `truncatedOnGap` is set + * (the defect) and nothing otherwise; for any other type it sends the + * conformant fresh `active{cursor, truncated: true}`. + */ + noReplay?: { name: string; truncatedOnGap?: boolean }; /** A per-type cap on streams, and optionally the control that reports it. */ quota?: QuotaBehaviour; /** @@ -413,7 +421,12 @@ export async function startEventsFixture( ); /** Open streams, so close() can tear them down instead of hanging on them. */ - const openStreams = new Set<{ res: ServerResponse; stop: () => void }>(); + const openStreams = new Set<{ + res: ServerResponse; + stop: () => void; + name: string; + notify: (method: string, params: Record) => void; + }>(); let liveStreams = 0; const liveByName = new Map(); @@ -463,6 +476,9 @@ export async function startEventsFixture( if (opts.quota && opts.quota.control !== false) { tools.push({ name: 'events_conformance_quota', inputSchema: obj }); } + if (opts.noReplay) { + tools.push({ name: 'events_conformance_yield_gap', inputSchema: obj }); + } if (opts.webhookEnvelopeControls?.gap) { tools.push({ name: 'events_conformance_webhook_gap', @@ -481,6 +497,32 @@ export async function startEventsFixture( } } + if ( + method === 'tools/call' && + opts.noReplay && + params.name === 'events_conformance_yield_gap' + ) { + const target = String( + ((params.arguments ?? {}) as Record).name ?? '' + ); + const isNoReplay = target === opts.noReplay.name; + for (const s of openStreams) { + if (s.name !== target) continue; + if (!isNoReplay) { + s.notify('notifications/events/active', { + cursor: 'cursor_after_gap', + truncated: true + }); + } else if (opts.noReplay.truncatedOnGap) { + s.notify('notifications/events/active', { + cursor: null, + truncated: true + }); + } + } + send({ content: [{ type: 'text', text: `ok: ${target}` }] }); + return; + } if ( method === 'tools/call' && opts.webhookEnvelopeControls && @@ -918,7 +960,13 @@ export async function startEventsFixture( liveStreams += 1; const key = String(name); if (capped) liveByName.set(key, (liveByName.get(key) ?? 0) + 1); - const entry = openStream(res, id, key, behaviour); + const entry = openStream( + res, + id, + key, + behaviour, + opts.noReplay?.name === key + ); openStreams.add(entry); const done = () => { if (!openStreams.delete(entry)) return; @@ -970,8 +1018,14 @@ function openStream( res: ServerResponse, requestId: unknown, name: string, - behaviour: StreamBehaviour & typeof CONFORMANT_STREAM -): { res: ServerResponse; stop: () => void } { + behaviour: StreamBehaviour & typeof CONFORMANT_STREAM, + noReplay = false +): { + res: ServerResponse; + stop: () => void; + name: string; + notify: (method: string, params: Record) => void; +} { const timers: NodeJS.Timeout[] = []; const stop = () => { for (const t of timers) clearInterval(t); @@ -1007,7 +1061,7 @@ function openStream( if (!behaviour.omitActive) { notify('notifications/events/active', { - cursor: 'cursor_stream_001', + cursor: noReplay ? null : 'cursor_stream_001', truncated: false, ...(behaviour.activeParams ?? {}) }); @@ -1079,7 +1133,7 @@ function openStream( }); } - return { res, stop }; + return { res, stop, name, notify }; } function isRecord(value: unknown): value is Record { diff --git a/src/scenarios/server/events/negative-push.test.ts b/src/scenarios/server/events/negative-push.test.ts index 22c656bd..fed92c99 100644 --- a/src/scenarios/server/events/negative-push.test.ts +++ b/src/scenarios/server/events/negative-push.test.ts @@ -374,6 +374,60 @@ describe.concurrent( } ); +describe.concurrent('truncated on a type without replay', () => { + const withNoReplay = ( + noReplay: EventsFixtureOptions['noReplay'] + ): EventsFixtureOptions => ({ + capability: { listChanged: true }, + descriptors: [ + descriptor({ name: 'push.event', delivery: ['push'] }), + descriptor({ name: 'quiet.event', delivery: ['push'] }) + ], + noReplay + }); + const id = 'sep-9999-truncated-false-when-no-replay'; + + test('a gap that sends nothing on a null-cursor stream passes', async () => { + const checks = await pushChecks(withNoReplay({ name: 'quiet.event' })); + const check = checks.get(id); + expect(check?.status).toBe('SUCCESS'); + expect(check?.details?.name).toBe('quiet.event'); + }); + + test('a gap that sends truncated:true on a null-cursor stream warns', async () => { + const checks = await pushChecks( + withNoReplay({ name: 'quiet.event', truncatedOnGap: true }) + ); + const check = checks.get(id); + expect(check?.status).toBe('WARNING'); + expect(check?.details?.untestable).toBeUndefined(); + expect(check?.details?.frame).toMatchObject({ + cursor: null, + truncated: true + }); + }); + + test('a catalog where every type replays makes the rule not apply', async () => { + const checks = await pushChecks(withNoReplay({ name: 'absent.event' })); + expect(checks.get(id)?.status).toBe('SKIPPED'); + }); + + test('without the gap control the row is untestable and names it', async () => { + const checks = await pushChecks(pushFixture()); + const check = checks.get(id); + expect(check?.details?.untestable).toBe(true); + expect(check?.errorMessage).toContain('events_conformance_yield_gap'); + }); + + test('an early return still reports the row', async () => { + const checks = await pushChecks({ + capability: { listChanged: true }, + listError: { code: -32603, message: 'boom' } + }); + expect(checks.get(id)?.details?.untestable).toBe(true); + }); +}); + describe.concurrent('concurrency and cancellation', () => { // The cap kitchen-sink applies to streams, which the document exempts them // from: the first stream confirms and the other two are refused -32013. diff --git a/src/scenarios/server/events/push.ts b/src/scenarios/server/events/push.ts index a1a400d6..1151312e 100644 --- a/src/scenarios/server/events/push.ts +++ b/src/scenarios/server/events/push.ts @@ -111,6 +111,19 @@ const STREAM_IDS = [ */ const ERROR_IDS = ['sep-9999-error-resource-exhausted'] as const; +/** + * Graded by poll too, where a replay-capable target makes it not applicable. + * Push can reach a type without replay whenever one offers push, so it grades + * the gap signal there: `truncated` SHOULD stay false with no position to have + * advanced past. + */ +const NO_REPLAY_ID = 'sep-9999-truncated-false-when-no-replay'; +const NO_REPLAY_DESCRIPTION = + 'For event types that do not support replay (`cursor` is always `null`), `truncated` SHOULD be `false`.'; + +/** How long to watch a no-replay stream after asking for a gap. */ +const NO_REPLAY_WATCH_MS = 1500; + function untestableAll( ids: readonly string[], reason: string, @@ -122,7 +135,7 @@ function untestableAll( } function skipAll(reason: string): ConformanceCheck[] { - return [...STREAM_IDS, ...ERROR_IDS].map((id) => + return [...STREAM_IDS, ...ERROR_IDS, NO_REPLAY_ID].map((id) => eventsCheck(id, id, 'SKIPPED', { errorMessage: reason }) ); } @@ -148,6 +161,25 @@ export class EventsPushScenario implements ClientScenario { **Untestable rather than green**: an upstream failure, a retention gap, a termination and a server-initiated close cannot be provoked from the client side, so those rows name the missing prerequisite instead of passing against a server that simply never did it.`; async run(ctx: RunContext): Promise { + const checks = await this.runStreams(ctx); + // Every path reports the no-replay row, so runs stay comparable: the early + // returns above the stream never reach the probe that grades it. + if (!checks.some((c) => c.id === NO_REPLAY_ID)) { + checks.push( + untestableCheck( + NO_REPLAY_ID, + NO_REPLAY_ID, + NO_REPLAY_DESCRIPTION, + 'No stream was opened, so no event type without replay was probed.', + [EVENTS_SPEC_REF], + 'WARNING' + ) + ); + } + return checks; + } + + private async runStreams(ctx: RunContext): Promise { const conn = await ctx.connect(); let declared = false; try { @@ -358,6 +390,7 @@ export class EventsPushScenario implements ClientScenario { // fallback for a server without the control; dedupe keeps the first. if (quota) checks.push(await this.quotaCheck(ctx, quota, descriptors)); checks.push(...(await this.concurrencyChecks(ctx, name, args))); + checks.push(await this.noReplayGapCheck(ctx, descriptors, controls.gap)); checks.push(...(await this.errorBeforeOpenChecks(ctx, args))); return dedupe(checks); } finally { @@ -365,6 +398,98 @@ export class EventsPushScenario implements ClientScenario { } } + /** + * `truncated-false-when-no-replay` on push: ask for a gap on a type whose + * stream confirms with `cursor: null`, and check no `active` frame arrives + * with `truncated: true`. + * + * Replay support is read off the stream's own `active`, the push analogue + * of poll reading `cursor: null` off its result, since `events/list` has no + * field that declares it. Each push type is opened in turn until one + * confirms with a null cursor; a catalog where every type replays makes the + * rule not applicable, as it is on poll. + */ + private async noReplayGapCheck( + ctx: RunContext, + descriptors: EventDescriptor[], + hasGap: boolean + ): Promise { + const untestable = (reason: string) => + untestableCheck( + NO_REPLAY_ID, + NO_REPLAY_ID, + NO_REPLAY_DESCRIPTION, + reason, + [EVENTS_SPEC_REF], + 'WARNING' + ); + if (!hasGap) { + return untestable( + `A gap cannot be provoked from the client side. Needs a fixture exposing the \`${EVENTS_CONTROL_YIELD_GAP}\` control for an event type without replay.` + ); + } + + let chosen: { name: string; session: StreamSession } | undefined; + for (const d of descriptors) { + const name = descriptorName(d); + const args = minimalArguments(d); + if (!name || args === undefined || !deliveryModes(d).includes('push')) + continue; + const session = await openEventStream( + ctx.serverUrl, + ctx.specVersion, + { name, arguments: args, cursor: null }, + { openTimeoutMs: ACTIVE_MS } + ); + const active = await session.waitFor( + (n) => n.method === EVENTS_ACTIVE_NOTIFICATION, + ACTIVE_MS + ); + if (active && active.params.cursor === null) { + chosen = { name, session }; + break; + } + await session.cancel(); + } + if (!chosen) { + return eventsCheck(NO_REPLAY_ID, NO_REPLAY_DESCRIPTION, 'SKIPPED', { + errorMessage: + 'Every push-capable event type confirmed its stream with a non-null cursor, so each supports replay and this rule does not apply.' + }); + } + + const { name, session } = chosen; + try { + const control = await ctx.connect(); + let fired: boolean; + try { + fired = await fireControl(control, EVENTS_CONTROL_YIELD_GAP, name); + } finally { + await control.close(); + } + if (!fired) { + return untestable( + `\`${name}\` confirms with \`cursor: null\`, but \`${EVENTS_CONTROL_YIELD_GAP}\` declined to signal a gap on it.` + ); + } + await session.settle(NO_REPLAY_WATCH_MS); + const truncated = session.notifications.find( + (n) => + n.method === EVENTS_ACTIVE_NOTIFICATION && n.params.truncated === true + ); + return truncated + ? eventsCheck(NO_REPLAY_ID, NO_REPLAY_DESCRIPTION, 'WARNING', { + errorMessage: `Event type \`${name}\` confirms its stream with \`cursor: null\` (no replay), and after a gap it sent \`${EVENTS_ACTIVE_NOTIFICATION}\` with \`truncated: true\`; there is no position to have advanced past.`, + details: { name, frame: truncated.params } + }) + : eventsCheck(NO_REPLAY_ID, NO_REPLAY_DESCRIPTION, 'SUCCESS', { + details: { name, watchedMs: NO_REPLAY_WATCH_MS } + }); + } finally { + await session.cancel(); + } + } + /** `notifications/events/active {cursor, truncated, _meta.subscriptionId}`. */ private activeCheck( active: { params: Record } | undefined,