Repository navigation
feat(models): add persistent per-model endpoint overrides #158
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -40,6 +40,13 @@ import { MODEL_COSTS, ZERO_MODEL_COST } from "./src/pricing.ts" | |
| import { registerCommandCodeQuota } from "./src/quota-command.ts" | ||
| import { normalizeCommandCodeResponsesResponse } from "./src/responses-stream.ts" | ||
| import { createCommandCodeRuntime } from "./src/runtime.ts" | ||
| import { | ||
| isEndpointChoice, | ||
| loadEndpointOverrides, | ||
| resolveEndpoint, | ||
| saveEndpointChoice, | ||
| type EndpointOverrides, | ||
| } from "./src/routing.ts" | ||
| import { createCommandCodeUsageProvider, type UsageProvider } from "./src/usage.ts" | ||
| import { transcriptReadersFrom, withTranscriptPromptAndTools } from "./src/transcript.ts" | ||
| import { createCommandCodeTransportRouter } from "./src/transport.ts" | ||
|
|
@@ -218,14 +225,55 @@ export default async function (pi: ExtensionAPI) { | |
| // The host dispatches every Command Code model through one custom api id, so | ||
| // the wire each model answers on is resolved here from the catalog that | ||
| // `createProviderConfig` registered. Unknown ids fall back to the id rule. | ||
| const catalogModels = new Map<string, CommandCodeModel>() | ||
| const routingPath = join(getAgentDir(), "commandcode-routing.json") | ||
| let routing: EndpointOverrides = {} | ||
| let routingWarning: string | undefined | ||
| try { | ||
| routing = await loadEndpointOverrides(routingPath) | ||
| } catch (error) { | ||
| routingWarning = error instanceof Error ? error.message : "Invalid routing configuration" | ||
| } | ||
| const resolveModelApi = (modelId: string): CommandCodeApi => { | ||
| if (routingWarning) throw new Error(routingWarning) | ||
| const model = catalogModels.get(modelId) | ||
| return model ? resolveEndpoint(model, routing) : apiForModelId(modelId) | ||
| } | ||
| pi.registerCommand("commandcode-endpoint", { | ||
| description: "Choose a persistent model endpoint: <model-id> auto|responses|chat-completions", | ||
| handler: async (args, ctx) => { | ||
| const [modelId, choice, extra] = args.trim().split(/\s+/) | ||
| const model = catalogModels.get(modelId ?? "") | ||
| if (!model || !isEndpointChoice(choice) || extra) { | ||
| ctx.ui.notify( | ||
| "Usage: /commandcode-endpoint <catalog-model-id> auto|responses|chat-completions", | ||
| "warning", | ||
| ) | ||
| return | ||
| } | ||
| await ctx.waitForIdle() | ||
| try { | ||
| resolveEndpoint(model, { [model.id]: choice }) | ||
| routing = await saveEndpointChoice(routingPath, model.id, choice) | ||
| routingWarning = undefined | ||
| ctx.ui.notify( | ||
| `Command Code endpoint for ${model.id}: ${resolveModelApi(model.id)} (${choice === "auto" ? "catalog" : "user override"}). Applies to the next request and survives refresh/restart.`, | ||
| "info", | ||
| ) | ||
| } catch (error) { | ||
| ctx.ui.notify( | ||
| error instanceof Error ? error.message : "Could not save routing configuration", | ||
| "error", | ||
| ) | ||
| } | ||
| }, | ||
| }) | ||
| let streamWarning: string | undefined | ||
| const transport = createCommandCodeTransportRouter({ | ||
| createStream: () => new AssistantMessageEventStream(), | ||
| streamProvider: (model, context, options) => { | ||
| const wire = resolveModelApi(model.id) | ||
| const catalogModel = catalogModels.get(model.id) | ||
| const resolvedOptions: Parameters<typeof streamNativeProvider>[2] = | ||
| resolveStreamOptions(options) | ||
| const nativeOptions = | ||
|
|
@@ -246,7 +294,12 @@ export default async function (pi: ExtensionAPI) { | |
| ...model, | ||
| api: wire, | ||
| cost: commandCodeCostRatesAt(model.id, model.cost), | ||
| compat: { | ||
| ...(model.compatConfig ?? model.compat), | ||
| // A selected host model can outlive a command or background refresh. | ||
| // Wire-specific defaults must follow the actual request, not that stale model. | ||
| ...(catalogModel ? compatForModel({ ...catalogModel, api: wire }) : {}), | ||
|
Owner
Author
There was a problem hiding this comment. Merging the host model's compat (computed for the catalog wire) with the compat for the overridden wire leaves wire-specific flags from the wrong wire in place. For a Responses model forced to Chat Completions,
compat: catalogModel
? compatForModel({ ...catalogModel, api: wire })
: (model.compatConfig ?? model.compat),(This is also the spot that overlaps with #162's |
||
| }, | ||
| }, | ||
| context, | ||
| nativeOptions, | ||
|
|
@@ -294,9 +347,19 @@ export default async function (pi: ExtensionAPI) { | |
| }), | ||
| loadCachedModels: () => loadCachedCommandCodeModels(modelsCachePath), | ||
| createProviderConfig: (models) => { | ||
| catalogModels.clear() | ||
| for (const model of models) catalogModels.set(model.id, model) | ||
| return createProviderConfig(models, apiBase, transport.stream) | ||
| }, | ||
| getRoutingStatus: (model) => { | ||
| if (routingWarning) return `routing warning: ${routingWarning}` | ||
| if (model?.provider !== "commandcode") return "model endpoint: no Command Code model selected" | ||
| try { | ||
| return `model endpoint: ${resolveModelApi(model.id)} (${routing[model.id] ?? "auto"}; ${routing[model.id] ? "user override" : "catalog"})` | ||
| } catch (error) { | ||
| return `routing warning: ${error instanceof Error ? error.message : "Invalid route"}` | ||
| } | ||
| }, | ||
| getTransport: transport.getTransport, | ||
| getStreamWarning: () => streamWarning, | ||
| }) | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| import { randomUUID } from "node:crypto" | ||
| import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises" | ||
| import { dirname } from "node:path" | ||
| import type { CommandCodeApi, CommandCodeModel } from "./models.ts" | ||
|
|
||
| export type EndpointChoice = "auto" | "responses" | "chat-completions" | ||
| export type EndpointOverrides = Readonly<Record<string, EndpointChoice>> | ||
|
|
||
| export function isEndpointChoice(value: unknown): value is EndpointChoice { | ||
| return value === "auto" || value === "responses" || value === "chat-completions" | ||
| } | ||
|
|
||
| export function parseEndpointOverrides(value: unknown): EndpointOverrides { | ||
| if (typeof value !== "object" || value === null || Array.isArray(value)) { | ||
| throw new Error("Routing configuration must be an object with version: 1 and models") | ||
| } | ||
| if (!("version" in value) || value.version !== 1 || !("models" in value)) { | ||
| throw new Error("Unsupported routing configuration version") | ||
| } | ||
| const models = value.models | ||
| if (typeof models !== "object" || models === null || Array.isArray(models)) { | ||
| throw new Error("Routing models must be an object") | ||
| } | ||
| const entries = Object.entries(models) | ||
| if (entries.some(([id, choice]) => !id.trim() || !isEndpointChoice(choice))) { | ||
| throw new Error("Routing choices must be auto, responses, or chat-completions") | ||
| } | ||
| return Object.fromEntries( | ||
|
Owner
Author
There was a problem hiding this comment.
|
||
| entries.filter((entry): entry is [string, EndpointChoice] => isEndpointChoice(entry[1])), | ||
| ) | ||
| } | ||
|
|
||
| export async function loadEndpointOverrides(path: string): Promise<EndpointOverrides> { | ||
| try { | ||
| const value: unknown = JSON.parse(await readFile(path, "utf8")) | ||
| return parseEndpointOverrides(value) | ||
| } catch (error) { | ||
| if (error instanceof Error && "code" in error && error.code === "ENOENT") return {} | ||
| // Do not echo file contents, JSON parse excerpts, paths, or arbitrary values. | ||
| throw new Error( | ||
| "Could not read Command Code routing configuration; check its version and model choices", | ||
|
Owner
Author
There was a problem hiding this comment. Two issues with the fail-closed path:
I'd either fail open (ignore the file, keep |
||
| ) | ||
| } | ||
| } | ||
|
|
||
| export function resolveEndpoint( | ||
| model: CommandCodeModel, | ||
| overrides: EndpointOverrides, | ||
| ): CommandCodeApi { | ||
| const choice = overrides[model.id] ?? "auto" | ||
| if (choice === "auto") return model.api | ||
| if (model.api === "anthropic-messages") { | ||
| throw new Error("Claude models require Anthropic Messages; use auto") | ||
| } | ||
| if (choice === "responses" && model.api !== "openai-responses") { | ||
| throw new Error( | ||
| "The current catalog does not advertise Responses for this model; refresh the catalog or use auto", | ||
| ) | ||
| } | ||
| return choice === "responses" ? "openai-responses" : "openai-completions" | ||
| } | ||
|
|
||
| /** The routing file is separate from the generated model cache and package contents. */ | ||
| export async function saveEndpointChoice( | ||
| path: string, | ||
| modelId: string, | ||
| choice: EndpointChoice, | ||
| ): Promise<EndpointOverrides> { | ||
| const overrides = { ...(await loadEndpointOverrides(path)) } | ||
| if (choice === "auto") delete overrides[modelId] | ||
| else | ||
| Object.defineProperty(overrides, modelId, { | ||
|
Owner
Author
There was a problem hiding this comment. A comment on why |
||
| value: choice, | ||
| enumerable: true, | ||
| configurable: true, | ||
| writable: true, | ||
| }) | ||
| await mkdir(dirname(path), { recursive: true }) | ||
| const temporary = `${path}.${randomUUID()}.tmp` | ||
| try { | ||
| await writeFile(temporary, JSON.stringify({ version: 1, models: overrides }, null, 2) + "\n", { | ||
| encoding: "utf8", | ||
| mode: 0o600, | ||
| flag: "wx", | ||
| }) | ||
| await rename(temporary, path) | ||
| } finally { | ||
| await rm(temporary, { force: true }) | ||
| } | ||
| return overrides | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Note for the fail-closed discussion on
src/routing.ts: this throw blocks every Provider API request for every model, including models that have no override at all. If you keep fail-closed, consider scoping it: a model without an entry in a parseable-but-partially-invalid file could still routeauto.