From 6647735cebb1b1803f069dbfd810775ba360feab Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 13:21:08 +0000 Subject: [PATCH] feat(client): load scenarios from outside the repository Add `--scenario-file ` to `client` and `list`. The file is a JavaScript module whose default export is one scenario or an array of them. They run through the same runner as the built-in scenarios, so a team can keep checks for its own product next to its code, or try a scenario out before proposing it here. Loaded scenarios stay apart from conformance: - with `--scenario-file`, only loaded scenarios can be selected, by `--scenario ` or as the `custom` suite; combining it with `--requirements`, a built-in suite or a built-in scenario is an error, raised before any file is imported - a name must be path-safe and must not match any built-in scenario - every run says which scenarios were loaded and that they are not part of MCP conformance; `list` and the suite summary mark them "(custom)" A loaded scenario is untyped, so the checks it returns are checked: a mistyped status fails the run instead of reading as a pass. `examples/scenarios/trace-id.mjs` is a complete example with a client that passes it, and the README documents the scenario object. --- README.md | 50 +++- examples/scenarios/trace-id-client.mjs | 19 ++ examples/scenarios/trace-id.mjs | 104 +++++++ src/index.ts | 61 +++- src/scenarios/custom.test.ts | 399 +++++++++++++++++++++++++ src/scenarios/custom.ts | 225 ++++++++++++++ src/scenarios/index.ts | 3 +- 7 files changed, 854 insertions(+), 7 deletions(-) create mode 100644 examples/scenarios/trace-id-client.mjs create mode 100644 examples/scenarios/trace-id.mjs create mode 100644 src/scenarios/custom.test.ts create mode 100644 src/scenarios/custom.ts diff --git a/README.md b/README.md index d5c4cb3f..a50dd02e 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,8 @@ npx @modelcontextprotocol/conformance client --command "" --scen - `--command` - The command to run your MCP client (can include flags) - `--scenario` - The test scenario to run (e.g., "initialize") -- `--suite` - Run a suite of tests in parallel: `all`, `core`, `extensions`, `backcompat`, `auth`, `metadata`, `draft` (scenarios targeting the in-progress draft spec), or `sep-835` +- `--suite` - Run a suite of tests in parallel: `all`, `core`, `extensions`, `backcompat`, `auth`, `metadata`, `draft` (scenarios targeting the in-progress draft spec), `custom` (the scenarios loaded with `--scenario-file`), or `sep-835` +- `--scenario-file ` - Load your own scenarios from a JavaScript module; repeatable (see [Writing Your Own Scenarios](#writing-your-own-scenarios)) - `--spec-version ` - Filter scenarios by spec version (e.g., `2025-11-25`, `2026-07-28`; `draft` is accepted as an alias for the current draft identifier). The draft version selects the latest dated release plus any draft-only scenarios. When omitted, the version is inferred from the scenario's spec applicability (draft-only scenarios run at the draft version, everything else at the latest dated release); an explicitly requested version outside a scenario's applicability window skips the scenario (exit 0) unless `--force` is passed - `--force` - Run a scenario even if it is not applicable at the requested `--spec-version` - `--requirements ` - Run exactly what a spec revision requires, frozen at its release (see [Conformance Requirements](#conformance-requirements)) @@ -256,6 +257,53 @@ Two things to know: A scenario cannot be listed both wholesale and per-check — the wholesale entry already excuses everything, so the pair is contradictory and is rejected. +## Writing Your Own Scenarios + +You can run client scenarios that are not part of this repository: checks that belong to your own product, or a scenario you are trying out before proposing it here. Put them in a JavaScript module and pass it with `--scenario-file`: + +```bash +npx @modelcontextprotocol/conformance client \ + --scenario-file ./my-scenarios.mjs \ + --command "" +``` + +[`examples/scenarios/trace-id.mjs`](./examples/scenarios/trace-id.mjs) is a complete, commented scenario, and [`trace-id-client.mjs`](./examples/scenarios/trace-id-client.mjs) is a client that passes it. From a checkout of this repository, after `npm install && npm run build`: + +```bash +node dist/index.js client \ + --scenario-file examples/scenarios/trace-id.mjs \ + --scenario example/trace-id \ + --command "node examples/scenarios/trace-id-client.mjs" +``` + +**This interface is experimental** and may change between releases. + +### The scenario object + +The module's default export is one scenario or an array of them. A scenario plays the server and judges what the client sent. It imports nothing from this package. + +| Member | What it is | +| ------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `name` | Parts of letters, digits, `.`, `_` and `-`, each starting with a letter or digit, with `/` between parts. Prefix it (`acme/login`) so that it cannot clash with a built-in scenario, now or later. | +| `description` | One sentence. | +| `source` | `{ introducedIn: '' }`, the first spec version the scenario applies to, and optionally `removedIn`. | +| `start(ctx)` | Starts the server and returns `{ serverUrl }`. `ctx.createServer(handlers)` serves one handler per method, `(params, request) => result`, which may be async, with the lifecycle of `ctx.specVersion` around them. An optional `context` object in the return value reaches the client as `MCP_CONFORMANCE_CONTEXT`. | +| `getChecks()` | Returns the checks, synchronously. It is called after the client has finished and before `stop()`. `server.recorded` holds every request and notification the client sent, in order, without the lifecycle messages. | +| `stop()` | Closes the server. | +| `allowClientError` | Optional. `true` if the client is expected to exit with an error. | + +A check has an `id`, a `name`, a `description`, a `timestamp` and a `status`: `SUCCESS`, `FAILURE`, `WARNING`, `SKIPPED` or `INFO`. `FAILURE` and `WARNING` fail the run. `errorMessage`, `details` and `specReferences` are optional. Use one `id` per check whether it passes or fails; expected-failures entries refer to it. Give your ids a prefix of your own: `traceability` counts ids of the form `sep--…`. The runner adds a `wire-schema-valid` check when it has seen traffic. More conventions are in [AGENTS.md](./AGENTS.md). + +### How loaded scenarios run + +- **Selection.** With `--scenario-file`, only the loaded scenarios can be selected. `--scenario ` runs one and prints each check; without `--command` it starts in interactive mode and prints the server URL, so any client can be pointed at it. Without `--scenario`, all loaded scenarios run as the `custom` suite. +- **Marked as custom.** Every run names the loaded scenarios and says that they are not part of MCP conformance. `list --scenario-file ` and the suite summary mark them `(custom)`. +- **Spec versions.** `--spec-version` selects and skips them by their `source`, as for built-in scenarios. The same scenario serves every spec version it applies to; the example client speaks the lifecycle of `2025-11-25`, the default. +- **Not part of conformance.** Loaded scenarios never count towards a requirement set or a tier: `--scenario-file` cannot be combined with `--requirements` or with a built-in suite. [Expected failures](#expected-failures) work for them as for any scenario. +- **Trust.** The module is loaded with `import()`, which runs its code. Load only files you trust. + +A scenario that has proved useful can be proposed for the suite: see [CONTRIBUTING.md](./CONTRIBUTING.md). + ## GitHub Action This repo provides a composite GitHub Action so SDK repos don't need to write their own conformance scripts. diff --git a/examples/scenarios/trace-id-client.mjs b/examples/scenarios/trace-id-client.mjs new file mode 100644 index 00000000..194542e2 --- /dev/null +++ b/examples/scenarios/trace-id-client.mjs @@ -0,0 +1,19 @@ +/** + * A client that passes examples/scenarios/trace-id.mjs. The runner appends + * the server URL to the command. It uses the SDK, so it speaks the lifecycle + * of spec version 2025-11-25, which is the default. + */ +import { argv } from 'node:process'; +import { URL } from 'node:url'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; + +const client = new Client({ name: 'trace-id-client', version: '1.0.0' }); +await client.connect(new StreamableHTTPClientTransport(new URL(argv[2]))); +await client.listTools(); +await client.callTool({ + name: 'echo', + arguments: { text: 'hello' }, + _meta: { 'com.example/traceId': 'trace-0001' } +}); +await client.close(); diff --git a/examples/scenarios/trace-id.mjs b/examples/scenarios/trace-id.mjs new file mode 100644 index 00000000..250d70e5 --- /dev/null +++ b/examples/scenarios/trace-id.mjs @@ -0,0 +1,104 @@ +/** + * Example of a scenario kept outside the suite: a check that belongs to one + * product, not to the spec. Here a team requires its client to tag every + * tool call with a trace id in `_meta`. The README, "Writing Your Own + * Scenarios", says how to run it. + * + * A scenario plays the server. `start()` serves the handlers below and + * returns the URL the client under test is given; `getChecks()` judges what + * the client sent. Nothing is imported from the suite. + */ + +const TRACE_ID = 'com.example/traceId'; + +const SPEC_REFERENCES = [ + { + id: 'MCP-Tools', + url: 'https://modelcontextprotocol.io/specification/2025-11-25/server/tools#calling-tools' + }, + { + id: 'MCP-Meta', + url: 'https://modelcontextprotocol.io/specification/2025-11-25/basic/index#_meta' + } +]; + +class TraceIdScenario { + // Prefix your names so they can never clash with a built-in scenario. + name = 'example/trace-id'; + description = 'Example: the client tags every tool call with a trace id'; + // The first spec version the scenario applies to. + source = { introducedIn: '2025-06-18' }; + server = null; + + async start(ctx) { + // One handler per method. It receives the request's `params` and returns + // the result. The suite supplies the lifecycle for `ctx.specVersion`. + this.server = await ctx.createServer({ + 'tools/list': () => ({ + tools: [ + { + name: 'echo', + description: 'Return the text it is given', + inputSchema: { + type: 'object', + properties: { text: { type: 'string' } }, + required: ['text'] + } + } + ] + }), + 'tools/call': (params) => ({ + content: [{ type: 'text', text: String(params.arguments?.text) }] + }) + }); + // `context`, if you return one, reaches the client as MCP_CONFORMANCE_CONTEXT. + return { serverUrl: this.server.url }; + } + + async stop() { + await this.server?.close(); + } + + // Called after the client has finished and before stop(). + getChecks() { + // Every request and notification the client sent, in order, without the + // lifecycle ones (initialize, notifications/initialized, server/discover). + const calls = (this.server?.recorded ?? []).filter( + (request) => request.method === 'tools/call' + ); + const untagged = calls.filter( + (call) => typeof call.params?._meta?.[TRACE_ID] !== 'string' + ); + const called = calls.length > 0; + const tagged = called && untagged.length === 0; + + // One id per check, whether it passes or fails. + return [ + { + id: 'example-tool-called', + name: 'ToolCalled', + description: 'The client called a tool', + status: called ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: SPEC_REFERENCES, + ...(!called && { errorMessage: 'The client never sent tools/call' }) + }, + { + id: 'example-trace-id-sent', + name: 'TraceIdSent', + description: `Every tool call carries _meta["${TRACE_ID}"]`, + status: tagged ? 'SUCCESS' : 'FAILURE', + timestamp: new Date().toISOString(), + specReferences: SPEC_REFERENCES, + ...(!tagged && { + errorMessage: called + ? `${untagged.length} of ${calls.length} tool calls had no trace id` + : 'The client never sent tools/call' + }) + } + ]; + } +} + +// Export one scenario, or an array of them. +export default new TraceIdScenario(); diff --git a/src/index.ts b/src/index.ts index b376644e..98b4302c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,12 @@ import { resolveSpecVersion } from './scenarios'; import type { SpecVersion } from './scenarios'; +import { + applyScenarioFiles, + isCustomScenario, + listCustomScenarios, + type ScenarioFileOptions +} from './scenarios/custom'; import { ConformanceCheck } from './types'; import { AuthorizationServerOptionsSchema, @@ -206,6 +212,31 @@ function filterScenariosBySpecVersion( return allScenarios.filter((s) => allowed.has(s)); } +function collect(value: string, previous: string[] = []): string[] { + return [...previous, value]; +} + +async function applyScenarioFilesOrExit( + options: ScenarioFileOptions +): Promise { + try { + const names = await applyScenarioFiles(options); + if (names.length > 0) { + console.error( + `Loaded custom scenarios, which are not part of MCP conformance: ${names.join(', ')}` + ); + } + return names; + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} + +function customLabel(name: string): string { + return isCustomScenario(name) ? ' (custom)' : ''; +} + const program = new Command(); program @@ -222,6 +253,11 @@ program .option('--command ', 'Command to run the client') .option('--scenario ', 'Scenario to test') .option('--suite ', 'Run a suite of tests in parallel (e.g., "auth")') + .option( + '--scenario-file ', + 'Load your own scenarios from a JavaScript module (repeatable). The module is executed.', + collect + ) .option('--timeout ', 'Timeout in milliseconds', '30000') .option( '--expected-failures ', @@ -243,6 +279,8 @@ program .option('--verbose', 'Show verbose output') .action(async (options, cmd) => { try { + const suiteGiven = options.suite !== undefined; + await applyScenarioFilesOrExit(options); const timeout = parseInt(options.timeout, 10); const verbose = options.verbose ?? false; const outputDir = options.outputDir; @@ -267,7 +305,11 @@ program // Handle suite mode if (options.suite || options.requirements !== undefined) { if (!options.command) { - console.error('--command is required when using --suite'); + console.error( + suiteGiven || requirements + ? '--command is required when using --suite' + : '--command is required to run the loaded scenarios. To start one in interactive mode, name it with --scenario.' + ); process.exit(1); } @@ -279,6 +321,7 @@ program auth: listAuthScenarios, metadata: listMetadataScenarios, draft: listDraftScenarios, + custom: listCustomScenarios, 'sep-835': () => listAuthScenarios().filter((name) => name.startsWith('auth/scope-')) }; @@ -393,7 +436,7 @@ program const status = failed === 0 && warnings === 0 ? '✓' : '✗'; const warningStr = warnings > 0 ? `, ${warnings} warnings` : ''; console.log( - `${status} ${result.scenario}: ${passed} passed, ${failed} failed${warningStr}` + `${status} ${result.scenario}${customLabel(result.scenario)}: ${passed} passed, ${failed} failed${warningStr}` ); if (verbose && failed > 0) { @@ -452,7 +495,7 @@ program console.error('\nAvailable client scenarios:'); listScenarios().forEach((s) => console.error(` - ${s}`)); console.error( - '\nAvailable suites: all, core, extensions, backcompat, auth, metadata, draft, sep-835' + '\nAvailable suites: all, core, extensions, backcompat, auth, metadata, draft, custom, sep-835' ); process.exit(1); } @@ -938,6 +981,11 @@ program .option('--client', 'List client scenarios') .option('--server', 'List server scenarios') .option('--authorization', 'List authorization server scenarios') + .option( + '--scenario-file ', + 'Also list the client scenarios a JavaScript module defines (repeatable). The module is executed.', + collect + ) .option( '--spec-version ', 'Filter scenarios by spec version (cumulative for date versions)' @@ -946,7 +994,8 @@ program '--requirements ', 'List exactly what a spec revision requires, frozen at its release' ) - .action((options) => { + .action(async (options) => { + const custom = await applyScenarioFilesOrExit(options); const specVersionFilter = options.specVersion ? resolveSpecVersion(options.specVersion) : undefined; @@ -995,7 +1044,7 @@ program } clientScenarioNames.forEach((s) => { const v = getScenarioSpecVersions(s); - console.log(` - ${s}${v ? ` [${v}]` : ''}`); + console.log(` - ${s}${v ? ` [${v}]` : ''}${customLabel(s)}`); }); } @@ -1023,6 +1072,8 @@ program console.log(` - ${s}${v ? ` [${v}]` : ''}`); }); } + // A loaded module may have left a timer or a socket open. + if (custom.length > 0) process.exit(0); }); program.parse(); diff --git a/src/scenarios/custom.test.ts b/src/scenarios/custom.test.ts new file mode 100644 index 00000000..4664216a --- /dev/null +++ b/src/scenarios/custom.test.ts @@ -0,0 +1,399 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { Client } from '@modelcontextprotocol/sdk/client/index.js'; +import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js'; +import { afterAll, beforeAll, describe, expect, test } from 'vitest'; +import { testScenarioContext } from '../mock-server/testing'; +import { runConformanceTest } from '../runner/client'; +import { LATEST_SPEC_VERSION } from '../types'; +import { + applyScenarioFiles, + isCustomScenario, + listCustomScenarios, + loadCustomScenarios +} from './custom'; +import { + getClientScenario, + getScenario, + getScenarioSpecVersions, + listAuthScenarios, + listBackcompatScenarios, + listCoreScenarios, + listDraftScenarios, + listExtensionScenarios, + listMetadataScenarios, + listScenariosForSpec +} from './index'; + +const EXAMPLE = 'examples/scenarios/trace-id.mjs'; +const EXAMPLE_NAME = 'example/trace-id'; +const EXAMPLE_CLIENT = 'node examples/scenarios/trace-id-client.mjs'; + +/** Source of a module that default-exports one scenario. */ +const scenarioSource = ( + name: string, + { + start = `return { serverUrl: 'http://127.0.0.1:1/mcp' };`, + checks = '[]' + } = {} +) => ` +export default { + name: ${JSON.stringify(name)}, + description: 'test scenario', + source: { introducedIn: '2025-11-25' }, + async start() { ${start} }, + async stop() {}, + getChecks() { return ${checks}; } +}; +`; + +let dir: string; +let files = 0; +const write = async (content: string, file = `file-${++files}.mjs`) => { + await writeFile(path.join(dir, file), content); + return path.join(dir, file); +}; + +beforeAll(async () => { + dir = await mkdtemp(path.join(tmpdir(), 'custom-scenarios-')); + await loadCustomScenarios([EXAMPLE]); +}); +afterAll(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +describe('loadCustomScenarios', () => { + test('registers the example scenario and marks it custom', () => { + expect(getScenario(EXAMPLE_NAME)?.description).toContain('Example'); + expect(isCustomScenario(EXAMPLE_NAME)).toBe(true); + expect(listCustomScenarios()).toContain(EXAMPLE_NAME); + expect(isCustomScenario('initialize')).toBe(false); + }); + + test('a loaded scenario follows --spec-version', () => { + expect(listScenariosForSpec(LATEST_SPEC_VERSION)).toContain(EXAMPLE_NAME); + expect(listScenariosForSpec('2025-03-26')).not.toContain(EXAMPLE_NAME); + }); + + test('a loaded scenario joins no built-in suite', () => { + const builtInSuites = [ + listCoreScenarios, + listExtensionScenarios, + listBackcompatScenarios, + listAuthScenarios, + listMetadataScenarios, + listDraftScenarios + ]; + for (const list of builtInSuites) { + expect(list()).not.toContain(EXAMPLE_NAME); + } + }); + + test('accepts an array of scenarios', async () => { + const file = await write( + `const make = (name) => ({ + name, + description: 'test scenario', + source: { introducedIn: '2025-11-25' }, + async start() { return { serverUrl: 'http://127.0.0.1:1/mcp' }; }, + async stop() {}, + getChecks() { return []; } +}); +export default [make('custom/a'), make('custom/b')];` + ); + expect(await loadCustomScenarios([file])).toEqual(['custom/a', 'custom/b']); + }); + + test.each([ + ['a client scenario', 'initialize'], + ['a client scenario in another case', 'Initialize'], + ['a server scenario', 'server-initialize'], + [ + 'an authorization server scenario', + 'authorization-server-metadata-endpoint' + ], + ['a scenario that is already loaded', EXAMPLE_NAME] + ])('refuses the name of %s', async (_label, name) => { + const client = getScenario('initialize'); + const versions = getScenarioSpecVersions('server-initialize'); + const file = await write(scenarioSource(name)); + await expect(loadCustomScenarios([file])).rejects.toThrow( + `the scenario name '${name}' is already in use` + ); + expect(getScenario('initialize')).toBe(client); + expect(getScenario('server-initialize')).toBeUndefined(); + expect(getClientScenario('server-initialize')).toBeDefined(); + expect(getScenarioSpecVersions('server-initialize')).toEqual(versions); + expect(isCustomScenario(name)).toBe(name === EXAMPLE_NAME); + }); + + test('refuses the same name in two files', async () => { + const a = await write(scenarioSource('custom/dup')); + const b = await write(scenarioSource('custom/dup'), 'dup-b.mjs'); + await expect(loadCustomScenarios([a, b])).rejects.toThrow( + /dup-b\.mjs: the scenario name 'custom\/dup' is already in use/ + ); + expect(getScenario('custom/dup')).toBeUndefined(); + }); + + test('registers nothing when any scenario is unusable', async () => { + const good = await write(scenarioSource('custom/good')); + const bad = await write( + `export default { name: 'custom/bad', description: 'no methods', source: { introducedIn: '2025-11-25' } };`, + 'bad.mjs' + ); + await expect(loadCustomScenarios([good, bad])).rejects.toThrow( + /bad\.mjs: scenario 1 has no `start\(\)` method/ + ); + expect(getScenario('custom/good')).toBeUndefined(); + }); + + test.each([ + ['no default export', `export const scenario = {};`, /no default export/], + ['an empty array', `export default [];`, /empty array/], + ['a non-object', `export default 'scenario';`, /is not an object/], + [ + 'an unknown spec version', + scenarioSource('custom/version').replace('2025-11-25', '2099-01-01'), + /needs a `source` of known spec versions/ + ], + [ + 'an extension id in place of a spec version', + scenarioSource('custom/extension').replace( + `introducedIn: '2025-11-25'`, + `extensionId: 'com.example/extension'` + ), + /needs a `source` of known spec versions/ + ], + [ + 'an extension id next to a spec version', + scenarioSource('custom/both').replace( + `introducedIn: '2025-11-25'`, + `introducedIn: '2025-11-25', extensionId: 'com.example/extension'` + ), + /needs a `source` of known spec versions/ + ], + [ + 'the second of two scenarios', + `const first = ${scenarioSource('custom/first').replace('export default', '').trim().replace(/;$/, '')}; +export default [first, { name: 'custom/second' }];`, + /scenario 2 has no `description`/ + ], + [ + 'a module that throws', + `throw new Error('boom');`, + /file-\d+\.mjs: could not be loaded: boom/ + ], + [ + 'a module with a syntax error', + `export default {`, + /file-\d+\.mjs: could not be loaded: / + ] + ])('refuses %s', async (_label, content, message) => { + const file = await write(content); + await expect(loadCustomScenarios([file])).rejects.toThrow(message); + }); + + test.each([ + '', + '../../escape', + 'a/../b', + '/absolute', + 'trailing/', + '.hidden', + '_private', + ' padded ', + 'acme:hello', + 'two\nlines', + 'colour\u001b[31m' + ])('refuses the name %j', async (name) => { + const file = await write(scenarioSource(name)); + await expect(loadCustomScenarios([file])).rejects.toThrow(/has the name /); + }); + + test('refuses a missing file and a directory', async () => { + const missing = path.join(dir, 'nope.mjs'); + await expect(loadCustomScenarios([missing])).rejects.toThrow( + /nope\.mjs: could not be loaded: / + ); + await expect(loadCustomScenarios([dir])).rejects.toThrow( + ': could not be loaded: ' + ); + }); +}); + +describe('checks returned by a loaded scenario', () => { + const load = async ( + name: string, + parts: Parameters[1] + ) => { + await loadCustomScenarios([await write(scenarioSource(name, parts))]); + return getScenario(name)!; + }; + const check = (fields: object) => + JSON.stringify([ + { + id: 'a-check', + name: 'ACheck', + description: 'a check', + status: 'SUCCESS', + timestamp: '2026-01-01T00:00:00.000Z', + ...fields + } + ]); + + test('a valid check passes through', async () => { + const scenario = await load('checked/valid', { checks: check({}) }); + expect(scenario.getChecks()).toMatchObject([ + { id: 'a-check', status: 'SUCCESS' } + ]); + }); + + test('a frozen array of checks can be extended by the runner', async () => { + const scenario = await load('checked/frozen', { + checks: `Object.freeze(${check({})})` + }); + expect(() => + scenario.getChecks().push(...scenario.getChecks()) + ).not.toThrow(); + }); + + test('allowClientError set in start() reaches the runner', async () => { + const scenario = await load('checked/allow', { + start: `this.allowClientError = true; return { serverUrl: 'http://127.0.0.1:1/mcp' };` + }); + expect(scenario.allowClientError).toBeUndefined(); + await scenario.start(testScenarioContext()); + expect(scenario.allowClientError).toBe(true); + }); + + test.each([ + [ + 'mistyped-status', + check({ status: 'PASS' }), + /check 1 has the status "PASS"; use one of SUCCESS, FAILURE/ + ], + ['missing-id', check({ id: undefined }), /check 1 has no `id`/], + ['not-an-object', `['SUCCESS']`, /check 1 is not an object/], + [ + 'a-promise', + `Promise.resolve([])`, + /getChecks\(\) must return an array of checks/ + ] + ])('getChecks() returning %s throws', async (label, checks, message) => { + const scenario = await load(`checked/${label}`, { checks }); + expect(() => scenario.getChecks()).toThrow(message); + }); +}); + +describe('applyScenarioFiles', () => { + type Options = Parameters[0]; + + test('does nothing without files', async () => { + const options: Options = { suite: 'core' }; + expect(await applyScenarioFiles(options)).toEqual([]); + expect(options).toEqual({ suite: 'core' }); + }); + + test('refuses --suite custom without files', async () => { + await expect(applyScenarioFiles({ suite: 'custom' })).rejects.toThrow( + '--suite custom needs at least one --scenario-file' + ); + }); + + test.each([ + [{ requirements: '2025-11-25' }, /cannot be combined with --requirements/], + [{ suite: 'all' }, /cannot be combined with --suite all/], + [{ suite: 'core' }, /cannot be combined with --suite core/], + [ + { scenario: 'initialize' }, + /cannot be combined with the built-in scenario 'initialize'/ + ] + ])('refuses %j before importing anything', async (selection, message) => { + const file = await write(`throw new Error('must not be imported');`); + await expect( + applyScenarioFiles({ scenarioFile: [file], ...selection }) + ).rejects.toThrow(message); + }); + + test('selects the custom suite when no scenario is named', async () => { + const file = await write(scenarioSource('apply/suite')); + const options: Options = { scenarioFile: [file] }; + expect(await applyScenarioFiles(options)).toEqual(['apply/suite']); + expect(options.suite).toBe('custom'); + }); + + test('keeps a named scenario that the files define', async () => { + const file = await write(scenarioSource('apply/named')); + const options: Options = { scenarioFile: [file], scenario: 'apply/named' }; + await applyScenarioFiles(options); + expect(options.suite).toBeUndefined(); + }); + + test('refuses a named scenario that the files do not define', async () => { + const file = await write(scenarioSource('apply/other')); + await expect( + applyScenarioFiles({ scenarioFile: [file], scenario: 'apply/missing' }) + ).rejects.toThrow( + '--scenario apply/missing is not one of the loaded scenarios: apply/other' + ); + }); +}); + +describe('the example scenario', () => { + const own = (checks: { id: string; status: string }[]) => + checks + .filter((c) => c.id.startsWith('example-')) + .map((c) => [c.id, c.status]); + + test('passes with the example client', async () => { + const result = await runConformanceTest( + EXAMPLE_CLIENT, + EXAMPLE_NAME, + 20000 + ); + expect(result.clientOutput?.exitCode).toBe(0); + expect(own(result.checks)).toEqual([ + ['example-tool-called', 'SUCCESS'], + ['example-trace-id-sent', 'SUCCESS'] + ]); + expect(result.checks.some((c) => c.status === 'FAILURE')).toBe(false); + }, 30000); + + test('fails with a client that does nothing', async () => { + const result = await runConformanceTest('node -e ""', EXAMPLE_NAME, 20000); + expect(own(result.checks)).toEqual([ + ['example-tool-called', 'FAILURE'], + ['example-trace-id-sent', 'FAILURE'] + ]); + }, 30000); + + test('fails only the trace id check when a call is untagged', async () => { + const scenario = getScenario(EXAMPLE_NAME)!; + const { serverUrl } = await scenario.start(testScenarioContext()); + try { + const client = new Client({ name: 'untagged', version: '1.0.0' }); + await client.connect( + new StreamableHTTPClientTransport(new URL(serverUrl)) + ); + await client.callTool({ + name: 'echo', + arguments: { text: 'a' }, + _meta: { 'com.example/traceId': 'trace-0001' } + }); + await client.callTool({ name: 'echo', arguments: { text: 'b' } }); + await client.close(); + expect(scenario.getChecks()).toMatchObject([ + { id: 'example-tool-called', status: 'SUCCESS' }, + { + id: 'example-trace-id-sent', + status: 'FAILURE', + errorMessage: '1 of 2 tool calls had no trace id' + } + ]); + } finally { + await scenario.stop(); + } + }); +}); diff --git a/src/scenarios/custom.ts b/src/scenarios/custom.ts new file mode 100644 index 00000000..79bda60f --- /dev/null +++ b/src/scenarios/custom.ts @@ -0,0 +1,225 @@ +/** + * Loads client-testing scenarios from files outside this repository, so a + * team can run its own scenarios through the same runner. See the README, + * "Writing Your Own Scenarios". + */ +import { pathToFileURL } from 'url'; +import { + isSpecVersion, + type CheckStatus, + type ConformanceCheck, + type Scenario +} from '../types'; +import { + getScenario, + listClientScenarios, + listClientScenariosForAuthorizationServer, + listScenarios, + registerScenario +} from './index'; + +const CHECK_STATUSES: readonly CheckStatus[] = [ + 'SUCCESS', + 'FAILURE', + 'WARNING', + 'SKIPPED', + 'INFO' +]; + +// Names become directory names, baseline keys and summary lines. +const NAME_PATTERN = /^[a-z0-9][a-z0-9._-]*(\/[a-z0-9][a-z0-9._-]*)*$/i; + +const customScenarioNames = new Set(); + +export function listCustomScenarios(): string[] { + return Array.from(customScenarioNames); +} + +export function isCustomScenario(name: string): boolean { + return customScenarioNames.has(name); +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null; +} + +/** Why `value` is not a usable scenario, or undefined when it is one. */ +function scenarioProblem(value: unknown): string | undefined { + if (!isRecord(value)) return 'is not an object'; + const { name, source } = value; + if (typeof name !== 'string' || !NAME_PATTERN.test(name)) { + return `has the name ${JSON.stringify(name)}; use parts of letters, digits, '.', '_' and '-' that start with a letter or digit, with '/' between parts`; + } + if (typeof value.description !== 'string') return 'has no `description`'; + for (const method of ['start', 'stop', 'getChecks']) { + if (typeof value[method] !== 'function') { + return `has no \`${method}()\` method`; + } + } + if ( + !isRecord(source) || + 'extensionId' in source || + !isSpecVersion(source.introducedIn) || + (source.removedIn !== undefined && !isSpecVersion(source.removedIn)) + ) { + return 'needs a `source` of known spec versions, `introducedIn` and optionally `removedIn`, and no `extensionId`'; + } + return undefined; +} + +/** Why `value` is not a usable check, or undefined when it is one. */ +function checkProblem(value: unknown): string | undefined { + if (!isRecord(value)) return 'is not an object'; + if (typeof value.id !== 'string' || value.id === '') return 'has no `id`'; + if (!CHECK_STATUSES.includes(value.status as CheckStatus)) { + return `has the status ${JSON.stringify(value.status)}; use one of ${CHECK_STATUSES.join(', ')}`; + } + return undefined; +} + +/** + * The scenario as the runner sees it. A loaded scenario is untyped, so the + * checks it returns are checked: a mistyped status must not read as a pass. + */ +function guarded(scenario: Scenario): Scenario { + return { + name: scenario.name, + description: scenario.description, + source: scenario.source, + get allowClientError() { + return scenario.allowClientError; + }, + start: (ctx) => scenario.start(ctx), + stop: async () => scenario.stop(), + getChecks(): ConformanceCheck[] { + const checks: unknown = scenario.getChecks(); + if (!Array.isArray(checks)) { + throw new Error( + `Scenario '${scenario.name}': getChecks() must return an array of checks` + ); + } + checks.forEach((check, index) => { + const problem = checkProblem(check); + if (problem !== undefined) { + throw new Error( + `Scenario '${scenario.name}': check ${index + 1} ${problem}` + ); + } + }); + return [...checks]; + } + }; +} + +async function importScenarios(file: string): Promise { + let module: unknown; + try { + module = await import(pathToFileURL(file).href); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + throw new Error(`${file}: could not be loaded: ${reason}`); + } + const exported = isRecord(module) ? module.default : undefined; + if (exported === undefined) { + throw new Error( + `${file}: no default export (export one scenario or an array of them)` + ); + } + const scenarios: unknown[] = Array.isArray(exported) ? exported : [exported]; + if (scenarios.length === 0) { + throw new Error(`${file}: the default export is an empty array`); + } + return scenarios; +} + +/** + * Import each file and register the scenarios it default-exports (one + * scenario or an array). Returns the registered names, in order. Throws, + * registering nothing, if any file or scenario is unusable. + */ +export async function loadCustomScenarios(files: string[]): Promise { + // No name may match a scenario of any command, whatever its case. + const taken = new Set( + [ + ...listScenarios(), + ...listClientScenarios(), + ...listClientScenariosForAuthorizationServer() + ].map((name) => name.toLowerCase()) + ); + const loaded: Scenario[] = []; + + for (const file of files) { + const candidates = await importScenarios(file); + candidates.forEach((candidate, index) => { + const problem = scenarioProblem(candidate); + if (problem !== undefined) { + throw new Error(`${file}: scenario ${index + 1} ${problem}`); + } + const scenario = candidate as Scenario; + if (taken.has(scenario.name.toLowerCase())) { + throw new Error( + `${file}: the scenario name '${scenario.name}' is already in use` + ); + } + taken.add(scenario.name.toLowerCase()); + loaded.push(scenario); + }); + } + + for (const scenario of loaded) { + registerScenario(scenario.name, guarded(scenario)); + customScenarioNames.add(scenario.name); + } + return loaded.map((scenario) => scenario.name); +} + +export interface ScenarioFileOptions { + scenarioFile?: string[]; + scenario?: string; + suite?: string; + requirements?: string; +} + +/** + * Apply `--scenario-file` to a command's options: load the files and, when + * no scenario is named, select the `custom` suite. With files given, only + * the scenarios they define can be selected; a conflicting option throws + * before any file is imported. + */ +export async function applyScenarioFiles( + options: ScenarioFileOptions +): Promise { + const files = options.scenarioFile ?? []; + const suite = options.suite?.toLowerCase(); + if (files.length === 0) { + if (suite === 'custom') { + throw new Error('--suite custom needs at least one --scenario-file'); + } + return []; + } + if (options.requirements !== undefined) { + throw new Error( + '--scenario-file cannot be combined with --requirements: a requirement set is fixed and never includes custom scenarios.' + ); + } + if (suite !== undefined && suite !== 'custom') { + throw new Error( + `--scenario-file cannot be combined with --suite ${options.suite}: loaded scenarios run as --suite custom or by --scenario .` + ); + } + if (options.scenario !== undefined && getScenario(options.scenario)) { + throw new Error( + `--scenario-file cannot be combined with the built-in scenario '${options.scenario}': only loaded scenarios can be selected.` + ); + } + + const names = await loadCustomScenarios(files); + if (options.scenario === undefined) { + options.suite = 'custom'; + } else if (!names.includes(options.scenario)) { + throw new Error( + `--scenario ${options.scenario} is not one of the loaded scenarios: ${names.join(', ')}` + ); + } + return names; +} diff --git a/src/scenarios/index.ts b/src/scenarios/index.ts index 375d7010..c4c815e3 100644 --- a/src/scenarios/index.ts +++ b/src/scenarios/index.ts @@ -487,7 +487,8 @@ export function isScenarioApplicableAt( } export function listScenariosForSpec(version: SpecVersion): string[] { - return scenariosList + // From the map, not the list, so scenarios added at run time are included. + return Array.from(scenarios.values()) .filter((s) => matchesSpecVersion(s.source, version)) .map((s) => s.name); }