From e3171d84b50899e24d6ed03c9da0d4c10c69f5eb Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 15 Oct 2026 22:51:31 +0200 Subject: [PATCH 1/2] feat(models): persist per-model endpoint choices --- CHANGELOG.md | 4 ++ README.md | 12 ++++++ index.ts | 73 ++++++++++++++++++++++++++++++--- src/routing.ts | 91 +++++++++++++++++++++++++++++++++++++++++ src/runtime.ts | 6 ++- tests/test-models.ts | 1 + tests/test-pi-local.mjs | 59 +++++++++++++++++++++++++- tests/test-routing.ts | 83 +++++++++++++++++++++++++++++++++++++ 8 files changed, 322 insertions(+), 7 deletions(-) create mode 100644 src/routing.ts create mode 100644 tests/test-routing.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ebcaee..85ca8c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Add `/commandcode-endpoint auto|responses|chat-completions` for persistent per-model Provider API routing. Choices survive catalog refreshes and restarts, are validated against the model wire, and are shown in `/commandcode-status` (#156). No automatic mid-stream endpoint switching is performed. + ## 0.7.7 - 2026-10-11 - Recognize the `off` reasoning effort now published for the DeepSeek V4 and V4.1 models, so the catalog parser no longer rejects it and the generate transport never forwards `off` as `reasoning_effort`. The Provider API and Oh My Pi adapters translate the level from the model's thinking metadata on their own wire. diff --git a/README.md b/README.md index 3000228..8428fa8 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,18 @@ The Provider API serves models on different wires, and `/provider/v1/models` adv A model whose catalog entry omits `supported_endpoints` keeps Chat Completions, which every non-Claude model serves, so discovery keeps working when the field is absent. The selection is applied per model, so a catalog that mixes wires streams each model correctly. +### Persistent endpoint overrides + +Use `/commandcode-endpoint chat-completions` to select Chat Completions for one model, for example: + +```txt +/commandcode-endpoint zai-org/GLM-5.3 chat-completions +``` + +Use `responses` to select Responses where the catalog advertises it, or `auto` to restore catalog routing. Claude models require `auto` (Anthropic Messages). The command waits for the current turn to finish and affects subsequent requests; it never retries a partial stream on another endpoint. `/commandcode-status` shows the effective model endpoint and whether it came from the catalog or a user override. The account transport (`provider` or `generate`) is independent; endpoint overrides apply only to Provider API requests. + +Choices are stored in `commandcode-routing.json` inside the host agent directory, separate from the generated model cache. They survive refreshes, restarts and package updates. Do not edit the model cache. A malformed routing file blocks Provider API requests rather than silently ignoring your selection; correct the file and reload. Changes made to the file externally require a reload. Concurrent commands from separate host processes are not coordinated; use one host to change routing settings. + ### Reasoning support Reasoning capability and selectable effort levels follow the official CLI catalog independently. Models can therefore be marked as reasoning-capable even when Command Code chooses their depth automatically. Models with explicit effort support register a model-specific `thinkingLevelMap`, so pi and OMP expose only valid levels. For a few reasoning models the CLI catalog ships no effort levels although the endpoint accepts `reasoning_effort`; `src/commandcode-catalog-overrides.ts` adds a manual level set for those on top of the generated catalog (currently empty because upstream publishes every level). The catalog sync removes an override as soon as upstream publishes its own levels. Pi's native OpenAI- and Anthropic-compatible providers translate the selected level for Provider API accounts; the existing Command Code generate transport sends the matching `reasoning_effort` for Go accounts. diff --git a/index.ts b/index.ts index da485c6..89a2cc9 100644 --- a/index.ts +++ b/index.ts @@ -40,6 +40,13 @@ import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" import { registerCommandCodeQuota } from "./src/quota-command.ts" import { normalizeCommandCodeResponsesResponse } from "./src/responses-stream.ts" import { createCommandCodeRuntime } from "./src/runtime.ts" +import { + isEndpointChoice, + loadEndpointOverrides, + resolveEndpoint, + saveEndpointChoice, + type EndpointOverrides, +} from "./src/routing.ts" import { createCommandCodeUsageProvider, type UsageProvider } from "./src/usage.ts" import { transcriptReadersFrom, withTranscriptPromptAndTools } from "./src/transcript.ts" import { createCommandCodeTransportRouter } from "./src/transport.ts" @@ -218,14 +225,55 @@ export default async function (pi: ExtensionAPI) { // The host dispatches every Command Code model through one custom api id, so // the wire each model answers on is resolved here from the catalog that // `createProviderConfig` registered. Unknown ids fall back to the id rule. - const modelApis = new Map() - const resolveModelApi = (modelId: string): CommandCodeApi => - modelApis.get(modelId) ?? apiForModelId(modelId) + const catalogModels = new Map() + const routingPath = join(getAgentDir(), "commandcode-routing.json") + let routing: EndpointOverrides = {} + let routingWarning: string | undefined + try { + routing = await loadEndpointOverrides(routingPath) + } catch (error) { + routingWarning = error instanceof Error ? error.message : "Invalid routing configuration" + } + const resolveModelApi = (modelId: string): CommandCodeApi => { + if (routingWarning) throw new Error(routingWarning) + const model = catalogModels.get(modelId) + return model ? resolveEndpoint(model, routing) : apiForModelId(modelId) + } + pi.registerCommand("commandcode-endpoint", { + description: "Choose a persistent model endpoint: auto|responses|chat-completions", + handler: async (args, ctx) => { + const [modelId, choice, extra] = args.trim().split(/\s+/) + const model = catalogModels.get(modelId ?? "") + if (!model || !isEndpointChoice(choice) || extra) { + ctx.ui.notify( + "Usage: /commandcode-endpoint auto|responses|chat-completions", + "warning", + ) + return + } + await ctx.waitForIdle() + try { + resolveEndpoint(model, { [model.id]: choice }) + routing = await saveEndpointChoice(routingPath, model.id, choice) + routingWarning = undefined + ctx.ui.notify( + `Command Code endpoint for ${model.id}: ${resolveModelApi(model.id)} (${choice === "auto" ? "catalog" : "user override"}). Applies to the next request and survives refresh/restart.`, + "info", + ) + } catch (error) { + ctx.ui.notify( + error instanceof Error ? error.message : "Could not save routing configuration", + "error", + ) + } + }, + }) let streamWarning: string | undefined const transport = createCommandCodeTransportRouter({ createStream: () => new AssistantMessageEventStream(), streamProvider: (model, context, options) => { const wire = resolveModelApi(model.id) + const catalogModel = catalogModels.get(model.id) const resolvedOptions: Parameters[2] = resolveStreamOptions(options) const nativeOptions = @@ -246,7 +294,12 @@ export default async function (pi: ExtensionAPI) { ...model, api: wire, cost: commandCodeCostRatesAt(model.id, model.cost), - compat: model.compatConfig ?? model.compat, + compat: { + ...(model.compatConfig ?? model.compat), + // A selected host model can outlive a command or background refresh. + // Wire-specific defaults must follow the actual request, not that stale model. + ...(catalogModel ? compatForModel({ ...catalogModel, api: wire }) : {}), + }, }, context, nativeOptions, @@ -294,9 +347,19 @@ export default async function (pi: ExtensionAPI) { }), loadCachedModels: () => loadCachedCommandCodeModels(modelsCachePath), createProviderConfig: (models) => { - for (const model of models) modelApis.set(model.id, model.api) + catalogModels.clear() + for (const model of models) catalogModels.set(model.id, model) return createProviderConfig(models, apiBase, transport.stream) }, + getRoutingStatus: (model) => { + if (routingWarning) return `routing warning: ${routingWarning}` + if (model?.provider !== "commandcode") return "model endpoint: no Command Code model selected" + try { + return `model endpoint: ${resolveModelApi(model.id)} (${routing[model.id] ?? "auto"}; ${routing[model.id] ? "user override" : "catalog"})` + } catch (error) { + return `routing warning: ${error instanceof Error ? error.message : "Invalid route"}` + } + }, getTransport: transport.getTransport, getStreamWarning: () => streamWarning, }) diff --git a/src/routing.ts b/src/routing.ts new file mode 100644 index 0000000..050c5de --- /dev/null +++ b/src/routing.ts @@ -0,0 +1,91 @@ +import { randomUUID } from "node:crypto" +import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises" +import { dirname } from "node:path" +import type { CommandCodeApi, CommandCodeModel } from "./models.ts" + +export type EndpointChoice = "auto" | "responses" | "chat-completions" +export type EndpointOverrides = Readonly> + +export function isEndpointChoice(value: unknown): value is EndpointChoice { + return value === "auto" || value === "responses" || value === "chat-completions" +} + +export function parseEndpointOverrides(value: unknown): EndpointOverrides { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Routing configuration must be an object with version: 1 and models") + } + if (!("version" in value) || value.version !== 1 || !("models" in value)) { + throw new Error("Unsupported routing configuration version") + } + const models = value.models + if (typeof models !== "object" || models === null || Array.isArray(models)) { + throw new Error("Routing models must be an object") + } + const entries = Object.entries(models) + if (entries.some(([id, choice]) => !id.trim() || !isEndpointChoice(choice))) { + throw new Error("Routing choices must be auto, responses, or chat-completions") + } + return Object.fromEntries( + entries.filter((entry): entry is [string, EndpointChoice] => isEndpointChoice(entry[1])), + ) +} + +export async function loadEndpointOverrides(path: string): Promise { + try { + const value: unknown = JSON.parse(await readFile(path, "utf8")) + return parseEndpointOverrides(value) + } catch (error) { + if (error instanceof Error && "code" in error && error.code === "ENOENT") return {} + // Do not echo file contents, JSON parse excerpts, paths, or arbitrary values. + throw new Error( + "Could not read Command Code routing configuration; check its version and model choices", + ) + } +} + +export function resolveEndpoint( + model: CommandCodeModel, + overrides: EndpointOverrides, +): CommandCodeApi { + const choice = overrides[model.id] ?? "auto" + if (choice === "auto") return model.api + if (model.api === "anthropic-messages") { + throw new Error("Claude models require Anthropic Messages; use auto") + } + if (choice === "responses" && model.api !== "openai-responses") { + throw new Error( + "The current catalog does not advertise Responses for this model; refresh the catalog or use auto", + ) + } + return choice === "responses" ? "openai-responses" : "openai-completions" +} + +/** The routing file is separate from the generated model cache and package contents. */ +export async function saveEndpointChoice( + path: string, + modelId: string, + choice: EndpointChoice, +): Promise { + const overrides = { ...(await loadEndpointOverrides(path)) } + if (choice === "auto") delete overrides[modelId] + else + Object.defineProperty(overrides, modelId, { + value: choice, + enumerable: true, + configurable: true, + writable: true, + }) + await mkdir(dirname(path), { recursive: true }) + const temporary = `${path}.${randomUUID()}.tmp` + try { + await writeFile(temporary, JSON.stringify({ version: 1, models: overrides }, null, 2) + "\n", { + encoding: "utf8", + mode: 0o600, + flag: "wx", + }) + await rename(temporary, path) + } finally { + await rm(temporary, { force: true }) + } + return overrides +} diff --git a/src/runtime.ts b/src/runtime.ts index afb0ba2..f7e5925 100644 --- a/src/runtime.ts +++ b/src/runtime.ts @@ -7,6 +7,7 @@ export interface CommandCodeUi { export interface CommandCodeCommandContext { ui: CommandCodeUi waitForIdle?: () => Promise + model?: { id: string; provider: string } } export interface CommandCodeRuntimeApi< @@ -31,6 +32,7 @@ export interface CommandCodeRuntimeOptions { loadCachedModels: () => Promise createProviderConfig: (models: readonly CommandCodeModel[]) => TProviderConfig getTransport?: () => "unknown" | "provider" | "generate" + getRoutingStatus?: (model: CommandCodeCommandContext["model"]) => string getStreamWarning?: () => string | undefined now?: () => number logWarning?: (message: string) => void @@ -311,7 +313,9 @@ export class CommandCodeRuntime { const status = this.getStatus() ctx.ui.notify( - formatCommandCodeStatus(status), + [formatCommandCodeStatus(status), this.options.getRoutingStatus?.(ctx.model)] + .filter(Boolean) + .join("\n"), status.warning || status.streamWarning ? "warning" : "info", ) }, diff --git a/tests/test-models.ts b/tests/test-models.ts index e66a492..c2a713e 100644 --- a/tests/test-models.ts +++ b/tests/test-models.ts @@ -1,3 +1,4 @@ +import "./test-routing.ts" import assert from "node:assert/strict" import { mkdtemp, mkdir, readFile, readdir, rm, utimes, writeFile } from "node:fs/promises" import { tmpdir } from "node:os" diff --git a/tests/test-pi-local.mjs b/tests/test-pi-local.mjs index 4fcf19a..fdcf6a8 100644 --- a/tests/test-pi-local.mjs +++ b/tests/test-pi-local.mjs @@ -6,7 +6,15 @@ import assert from "node:assert/strict" import { spawn, spawnSync } from "node:child_process" -import { accessSync, constants, mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs" +import { + accessSync, + constants, + mkdtempSync, + rmSync, + mkdirSync, + writeFileSync, + readFileSync, +} from "node:fs" import { createServer } from "node:http" import { tmpdir } from "node:os" import { delimiter, dirname, join, resolve } from "node:path" @@ -656,6 +664,22 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { await new Promise((resolve) => setTimeout(resolve, 100)) } + send({ + id: "endpoint-choice", + type: "prompt", + message: `/commandcode-endpoint ${TEST_MODEL} chat-completions`, + }) + await waitFor( + (event) => + event.type === "extension_ui_request" && + event.message?.includes("Applies to the next request"), + ) + assert.equal( + JSON.parse(readFileSync(join(agentDir, "commandcode-routing.json"), "utf8")).models[ + TEST_MODEL + ], + "chat-completions", + ) includeRefreshedModel = true send({ id: "refresh", type: "prompt", message: "/commandcode-refresh" }) await waitFor((event) => event.type === "response" && event.id === "refresh" && event.success) @@ -679,6 +703,16 @@ async function runRpcExtensionCommands(timeoutMs = 30_000) { event.message.includes("model count: 4"), ) + assert.match(statusAfter.message, /model endpoint: openai-completions.*user override/) + send({ + id: "endpoint-auto", + type: "prompt", + message: `/commandcode-endpoint ${TEST_MODEL} auto`, + }) + await waitFor( + (event) => + event.type === "extension_ui_request" && event.message?.includes("(catalog). Applies"), + ) return { commandNames, statusBefore: statusBefore.message, @@ -1167,6 +1201,29 @@ try { assert.equal(lastRequestBody?.stream, true) assert.ok(Array.isArray(lastRequestBody?.input), "Responses request should use the input array") + console.log( + "[pi-local] persisted endpoint override survives process restart and live catalog refresh", + ) + writeFileSync( + join(agentDir, "commandcode-routing.json"), + JSON.stringify({ version: 1, models: { [RESPONSES_TEST_MODEL]: "chat-completions" } }), + ) + const overridden = await runPi([ + "--no-extensions", + "-e", + EXT_PATH, + "-p", + "say mock token", + "--provider", + "commandcode", + "--model", + RESPONSES_TEST_MODEL, + ]) + assert.equal(overridden.code, 0, overridden.stderr) + assert.equal(lastRequestPath, "/provider/v1/chat/completions") + assert.ok(Array.isArray(lastRequestBody.messages)) + rmSync(join(agentDir, "commandcode-routing.json")) + console.log("[pi-local] Responses reasoning deltas stream before the item completes") requestCount = 0 let releaseResponsesReasoningGate diff --git a/tests/test-routing.ts b/tests/test-routing.ts new file mode 100644 index 0000000..fdccb4c --- /dev/null +++ b/tests/test-routing.ts @@ -0,0 +1,83 @@ +import assert from "node:assert/strict" +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { describe, it } from "node:test" +import { + loadEndpointOverrides, + parseEndpointOverrides, + resolveEndpoint, + saveEndpointChoice, +} from "../src/routing.ts" +import type { CommandCodeModel } from "../src/models.ts" + +const model: CommandCodeModel = { + id: "zai-org/GLM-5.3", + name: "GLM", + api: "openai-responses", + reasoning: true, + contextWindow: 1000000, + maxTokens: 65536, +} + +describe("persistent endpoint choices", () => { + it("uses catalog auto and per-model overrides without changing other models", () => { + assert.equal(resolveEndpoint(model, {}), "openai-responses") + assert.equal(resolveEndpoint(model, { [model.id]: "chat-completions" }), "openai-completions") + assert.equal( + resolveEndpoint({ ...model, id: "meta/muse-spark-1.3" }, { [model.id]: "chat-completions" }), + "openai-responses", + ) + assert.equal(resolveEndpoint({ ...model, api: "openai-completions" }, {}), "openai-completions") + assert.throws( + () => resolveEndpoint({ ...model, api: "openai-completions" }, { [model.id]: "responses" }), + /does not advertise/, + ) + assert.throws( + () => + resolveEndpoint( + { ...model, api: "anthropic-messages" }, + { [model.id]: "chat-completions" }, + ), + /Claude/, + ) + }) + it("survives restart and catalog replacement; auto removes just one override", async () => { + const dir = await mkdtemp(join(tmpdir(), "cc-routing-")) + const path = join(dir, "commandcode-routing.json") + try { + assert.deepEqual(await loadEndpointOverrides(path), {}) + await saveEndpointChoice(path, "other", "responses") + await saveEndpointChoice(path, model.id, "chat-completions") + assert.equal( + resolveEndpoint({ ...model }, await loadEndpointOverrides(path)), + "openai-completions", + ) + await saveEndpointChoice(path, model.id, "auto") + assert.deepEqual(await loadEndpointOverrides(path), { other: "responses" }) + assert.match(await readFile(path, "utf8"), /"version": 1/) + await writeFile(path, '{"secret":"sensitive", broken') + await assert.rejects( + loadEndpointOverrides(path), + (error: unknown) => error instanceof Error && !error.message.includes("sensitive"), + ) + await assert.rejects(saveEndpointChoice(path, model.id, "auto")) + assert.match(await readFile(path, "utf8"), /sensitive/) + } finally { + await rm(dir, { recursive: true, force: true }) + } + }) + it("validates file shapes without exposing arbitrary values", () => { + for (const value of [ + null, + [], + { version: 2, models: {} }, + { version: 1, models: { foo: "secret" } }, + ]) { + assert.throws(() => parseEndpointOverrides(value)) + } + assert.deepEqual(parseEndpointOverrides({ version: 1, models: { foo: "auto" } }), { + foo: "auto", + }) + }) +}) From 2b41afdb75f65ad7bf5cd9e3c0e86448f2ac1410 Mon Sep 17 00:00:00 2001 From: Patrick Wozniak Date: Thu, 15 Oct 2026 23:31:53 +0200 Subject: [PATCH 2/2] test(models): include routing regressions in default suites --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index c9ae9bb..9ad72cb 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "LICENSE" ], "scripts": { - "test": "npm run typecheck && node --test tests/test-release.mjs tests/test-pi-child-cleanup.mjs && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", + "test": "npm run typecheck && npm run test:routing && node --test tests/test-release.mjs tests/test-pi-child-cleanup.mjs && tsx tests/test-package-manifest.ts && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts && node tests/test-pi-isolated.mjs && node tests/test-pi-authenticated.mjs && node tests/test-pi-local.mjs && node tests/test-omp-compat.mjs", "typecheck": "tsc --noEmit", "format:check": "prettier --check '**/*.{ts,mjs,json,md}'", "format": "prettier --write '**/*.{ts,mjs,json,md}'", @@ -40,9 +40,10 @@ "sync:commandcode-catalog": "tsx .github/scripts/check-commandcode-model-metadata.ts command-code@latest --write", "test:quota": "tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts", "test:usage": "tsx tests/test-usage.ts", - "test:unit": "tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", + "test:unit": "npm run test:routing && tsx tests/test-api-key.ts && tsx tests/test-pure-functions.ts && tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts && tsx tests/test-runtime.ts && tsx tests/test-pricing.ts && tsx tests/test-pricing-check.ts && tsx tests/test-cost.ts && tsx tests/test-oauth.ts && tsx tests/test-abort.ts && tsx tests/test-overflow.ts && tsx tests/test-stream.ts && tsx tests/test-quota.ts && tsx tests/test-quota-command.ts && tsx tests/test-usage.ts && tsx tests/test-retry.ts && tsx tests/test-transport.ts", "test:api-key": "tsx tests/test-api-key.ts", "test:models": "tsx tests/test-models.ts && tsx tests/test-model-metadata-check.ts", + "test:routing": "tsx tests/test-routing.ts", "test:runtime": "tsx tests/test-runtime.ts", "test:pricing": "tsx tests/test-pricing.ts", "test:pricing-check": "tsx tests/test-pricing-check.ts",