diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 3cae671a466..3126cb4d3ed 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -215,6 +215,7 @@ Providers can expose a built-in shorthand, such as `agy` for `google-antigravity | `retryOnReset?` | `{ enabled?: boolean; replacements?: number }` | Native `openai-responses` providers, including `authMode: "forward"`. Opt-in replacement of a send that failed while the caller had observed nothing: absent means off, object presence enables it unless `enabled: false`. Covers both ambiguous stages — a connection that died before any response header, and an SSE body that died after the header while carrying only control events. Only a self-contained request is ever replaced: `store: false`, complete `input`, no `previous_response_id`, `conversation` or `stream_id`, and only client-executed tools. `replacements` is the number of replacement sends ONE logical request may make across every leg and every combo child (1..2, default 1) — not a per-leg retry count and not a send budget, so a replacement still has to fit inside the send allowance the leg already had. A request that already emitted output or a tool call is never replaced, whatever this is set to. The replacement inference may still be billed if the origin had already started the first one, which is why this is off by default. | | `autoToolChoiceOnlyModels?` | `string[]` | Models whose `tool_choice` accepts only `auto` or `none`; forced choices are downgraded. | | `preserveReasoningContentModels?` | `string[]` | Models requiring prior assistant `reasoning_content` in chat history. | +| `inlineThinkTagModels?` | `string[]` | `openai-chat` models served by a gateway that runs no server-side reasoning parser, so the model leaves its chain of thought inline in `content` as `` / `` / `` blocks and sends no `reasoning_content` or `reasoning_details`. Without this the whole chain of thought renders as the answer. Listed models have those blocks split back into reasoning on both the streamed and non-streamed paths. Off by default, and engaged only for a response that opens with a thinking tag, so a model that merely mentions a think tag inside an answer is never rewritten. Prefer a provider-side parser or `reasoningSplitModels` when the upstream supports either. | | `reasoningDetailsModels?` | `string[]` | Models whose endpoint returns thinking as a structured `reasoning_details` array (MiniMax M-series with `reasoning_split`); stream deltas are cumulative snapshots that are prefix-diffed, and preserved reasoning replays as a `reasoning_details` array instead of a `reasoning_content` string. | | `requiresReasoningPlaceholderModels?` | `string[]` | Models whose upstream rejects a tool_call continuation missing `reasoning_content` (DeepSeek thinking mode); a minimal placeholder is injected when the replay cache misses. Defaults to `preserveReasoningContentModels`; set `[]` to opt out. | | `showThinkingSummary?` | `boolean` | Display provider-authored summaries when a Responses client omits `reasoning.summary`. Explicit wire `"none"` wins; a client that serializes its preference as omission cannot be distinguished. Raw reasoning remains content and is never relabeled as a summary. The `google-antigravity` preset defaults to `true`; explicit `false` disables that default. CCA Gemini requests also opt into `generationConfig.thinkingConfig.includeThoughts` when display is enabled; image, Claude and gpt-oss requests do not. This does not change client configuration or global catalog summary defaults. | diff --git a/scripts/test-layout/layout.json b/scripts/test-layout/layout.json index b0a1b3b81ea..8ee1e5bb941 100644 --- a/scripts/test-layout/layout.json +++ b/scripts/test-layout/layout.json @@ -1072,6 +1072,7 @@ "openai-chat-eof.test.ts": "adapters/openai", "openai-chat-hardening.test.ts": "adapters/openai", "openai-chat-image-normalization.test.ts": "adapters/openai", + "openai-chat-inline-think-tags.test.ts": "adapters/openai", "openai-chat-invalid-tool-call-diagnostics.test.ts": "adapters/openai", "openai-chat-model-suffix.test.ts": "adapters/openai", "openai-chat-native-policy.test.ts": "adapters/openai", diff --git a/src/adapters/inline-think-tags.ts b/src/adapters/inline-think-tags.ts new file mode 100644 index 00000000000..663eb49a3e2 --- /dev/null +++ b/src/adapters/inline-think-tags.ts @@ -0,0 +1,233 @@ +import type { AdapterEvent } from "../types"; +import { modelInList } from "../types"; +import type { TranslatorBudget } from "../lib/translator-budget"; + +type ThinkingTag = "" | "" | ""; +type ParserState = "pre" | "thinking" | "scanning" | "streaming"; + +const OPEN_TAGS: ThinkingTag[] = ["", "", ""]; +const MAX_OPEN_TAG = Math.max(...OPEN_TAGS.map(t => t.length)); +const MAX_CLOSE_TAG = Math.max(...OPEN_TAGS.map(t => ` tag.startsWith(text) && text.length < tag.length); +} + +/** Move a send boundary back one unit rather than splitting a surrogate pair into U+FFFD. */ +function surrogateSafeCut(text: string, cut: number): number { + if (cut <= 0 || cut >= text.length) return Math.max(0, Math.min(cut, text.length)); + const atCut = text.charCodeAt(cut - 1); + return atCut >= 0xd800 && atCut <= 0xdbff ? cut - 1 : cut; +} + +export interface InlineThinkTagOptions { + /** + * Keep scanning for further think blocks after the first one closes. Kiro emits a single + * leading block, so it leaves this off and streams the rest verbatim. MiniMax M-series + * interleaves several blocks with answer segments, so a reusing adapter opts in. + */ + interleaved?: boolean; +} + +/** + * Recovers thinking that a gateway left inline in visible content as `` blocks instead of + * a separate `reasoning_content` / `reasoning_details` field. Shared by the Kiro adapter and by + * the openai-chat adapter's opt-in `inlineThinkTagModels`. + */ +export class InlineThinkTagParser { + private state: ParserState = "pre"; + private preBuffer = ""; + private thinkingBuffer = ""; + private closeTag = ""; + + private readonly interleaved: boolean; + private sawAnswerText = false; + + constructor(private readonly budget?: TranslatorBudget, options?: InlineThinkTagOptions) { + this.interleaved = options?.interleaved === true; + } + + private replaceCarry(field: "preBuffer" | "thinkingBuffer", next: string): void { + const previous = this[field]; + if (previous === next) return; + const previousBytes = Buffer.byteLength(previous); + const nextBytes = Buffer.byteLength(next); + const reservation = this.budget?.reserveTransient(nextBytes, { kind: "reasoning" }); + this[field] = next; + reservation?.commitRetained(); + this.budget?.releaseRetained(previousBytes, { kind: "reasoning" }); + } + + feed(text: string): AdapterEvent[] { + if (!text) return []; + if (this.state === "streaming") return [{ type: "text_delta", text }]; + if (this.state === "thinking") { + this.replaceCarry("thinkingBuffer", this.thinkingBuffer + text); + return this.drainThinking(); + } + if (this.state === "scanning") { + this.replaceCarry("preBuffer", this.preBuffer + text); + return this.drainScanning(); + } + this.replaceCarry("preBuffer", this.preBuffer + text); + const stripped = this.preBuffer.trimStart(); + const openTag = OPEN_TAGS.find(tag => stripped.startsWith(tag)); + if (openTag) { + this.state = "thinking"; + this.closeTag = closeTagFor(openTag); + this.replaceCarry("thinkingBuffer", stripped.slice(openTag.length)); + this.replaceCarry("preBuffer", ""); + return this.drainThinking(); + } + if (stripped.length <= MAX_OPEN_TAG && isPossibleOpenTagPrefix(stripped)) return []; + this.state = "streaming"; + const out = this.preBuffer; + this.replaceCarry("preBuffer", ""); + return out ? [{ type: "text_delta", text: out }] : []; + } + + flush(): AdapterEvent[] { + if (this.state === "thinking") { + const out = this.thinkingBuffer; + this.replaceCarry("thinkingBuffer", ""); + this.state = "streaming"; + return out ? [{ type: "reasoning_raw_delta", text: out }] : []; + } + if (this.preBuffer) { + const out = this.preBuffer; + this.replaceCarry("preBuffer", ""); + this.state = "streaming"; + return [{ type: "text_delta", text: out }]; + } + return []; + } + + /** Release any partial tag/content carry when the owning stream stops early. */ + dispose(): void { + this.replaceCarry("preBuffer", ""); + this.replaceCarry("thinkingBuffer", ""); + this.closeTag = ""; + this.state = "streaming"; + } + + private drainThinking(): AdapterEvent[] { + const close = this.closeTag; + const idx = this.thinkingBuffer.indexOf(close); + if (idx >= 0) { + const thinking = this.thinkingBuffer.slice(0, idx); + const remainder = this.thinkingBuffer.slice(idx + close.length); + // The blank line a model leaves between its leading block and the answer is formatting + // noise, so it goes. Once the answer has started, whitespace is the answer's own: a + // mid-answer block sits inside markdown or code where indentation is meaningful. + const after = this.sawAnswerText ? remainder : remainder.trimStart(); + this.replaceCarry("thinkingBuffer", ""); + const events: AdapterEvent[] = []; + if (thinking) events.push({ type: "reasoning_raw_delta", text: thinking }); + if (this.interleaved) { + this.state = "scanning"; + this.replaceCarry("preBuffer", after); + events.push(...this.drainScanning()); + } else { + this.state = "streaming"; + if (after) events.push({ type: "text_delta", text: after }); + } + return events; + } + if (this.thinkingBuffer.length <= MAX_CLOSE_TAG) return []; + // Hold back a possible partial close tag, and never split a surrogate pair + // at the send boundary: a lone high surrogate encodes as U+FFFD. + const cut = surrogateSafeCut(this.thinkingBuffer, this.thinkingBuffer.length - MAX_CLOSE_TAG); + const send = this.thinkingBuffer.slice(0, cut); + this.replaceCarry("thinkingBuffer", this.thinkingBuffer.slice(cut)); + return send ? [{ type: "reasoning_raw_delta", text: send }] : []; + } + + /** + * Interleaved mode only: the response already proved it carries inline thinking, so a later + * block can open anywhere in the answer text rather than only at the start. + */ + private drainScanning(): AdapterEvent[] { + const events: AdapterEvent[] = []; + for (;;) { + let openIndex = -1; + let openTag: ThinkingTag | undefined; + for (const tag of OPEN_TAGS) { + const index = this.preBuffer.indexOf(tag); + if (index >= 0 && (openIndex < 0 || index < openIndex)) { + openIndex = index; + openTag = tag; + } + } + if (openIndex >= 0 && openTag) { + const before = this.preBuffer.slice(0, openIndex); + if (before) { this.sawAnswerText = true; events.push({ type: "text_delta", text: before }); } + this.state = "thinking"; + this.closeTag = closeTagFor(openTag); + this.replaceCarry("thinkingBuffer", this.preBuffer.slice(openIndex + openTag.length)); + this.replaceCarry("preBuffer", ""); + events.push(...this.drainThinking()); + // drainThinking returns to "scanning" only when that block closed inside this chunk. + if ((this.state as ParserState) !== "scanning") return events; + continue; + } + // Hold back only as much as a partial open tag could occupy. + const cut = surrogateSafeCut(this.preBuffer, this.preBuffer.length - (MAX_OPEN_TAG - 1)); + if (cut > 0) { + this.sawAnswerText = true; + events.push({ type: "text_delta", text: this.preBuffer.slice(0, cut) }); + this.replaceCarry("preBuffer", this.preBuffer.slice(cut)); + } + return events; + } + } +} + +/** Visible-content splitter the openai-chat adapter holds for the life of one response. */ +export interface InlineThinkContentSplitter { + feed(text: string): AdapterEvent[]; + flush(): AdapterEvent[]; + dispose(): void; +} + +const PASSTHROUGH: InlineThinkContentSplitter = { + feed: text => [{ type: "text_delta", text }], + flush: () => [], + dispose: () => { /* nothing carried */ }, +}; + +/** + * Opt-in recovery for `inlineThinkTagModels`. A model that is not listed gets a passthrough that + * never inspects or rewrites visible content, so the 66 registry providers sharing the openai-chat + * adapter keep byte-exact behavior. + */ +export function createInlineThinkContentSplitter( + models: string[] | undefined, + modelId: string | undefined, + budget?: TranslatorBudget, +): InlineThinkContentSplitter { + if (!modelInList(models, modelId ?? "")) return PASSTHROUGH; + const parser = new InlineThinkTagParser(budget, { interleaved: true }); + return { + // An empty content delta stays an empty delta: it is a wire signal, not thinking. + feed: text => (text.length === 0 ? [{ type: "text_delta", text }] : parser.feed(text)), + flush: () => parser.flush(), + dispose: () => parser.dispose(), + }; +} + +/** One-shot form for a non-streaming response body. */ +export function splitInlineThinkContent( + models: string[] | undefined, + modelId: string | undefined, + budget: TranslatorBudget | undefined, + content: string, +): AdapterEvent[] { + const splitter = createInlineThinkContentSplitter(models, modelId, budget); + const events = [...splitter.feed(content), ...splitter.flush()]; + splitter.dispose(); + return events; +} diff --git a/src/adapters/kiro-thinking.ts b/src/adapters/kiro-thinking.ts deleted file mode 100644 index ee144e28783..00000000000 --- a/src/adapters/kiro-thinking.ts +++ /dev/null @@ -1,112 +0,0 @@ -import type { AdapterEvent } from "../types"; -import type { TranslatorBudget } from "../lib/translator-budget"; - -type ThinkingTag = "" | "" | ""; -type ParserState = "pre" | "thinking" | "streaming"; - -const OPEN_TAGS: ThinkingTag[] = ["", "", ""]; -const MAX_OPEN_TAG = Math.max(...OPEN_TAGS.map(t => t.length)); -const MAX_CLOSE_TAG = Math.max(...OPEN_TAGS.map(t => ` tag.startsWith(text) && text.length < tag.length); -} - -export class KiroThinkingParser { - private state: ParserState = "pre"; - private preBuffer = ""; - private thinkingBuffer = ""; - private closeTag = ""; - - constructor(private readonly budget?: TranslatorBudget) {} - - private replaceCarry(field: "preBuffer" | "thinkingBuffer", next: string): void { - const previous = this[field]; - if (previous === next) return; - const previousBytes = Buffer.byteLength(previous); - const nextBytes = Buffer.byteLength(next); - const reservation = this.budget?.reserveTransient(nextBytes, { kind: "reasoning" }); - this[field] = next; - reservation?.commitRetained(); - this.budget?.releaseRetained(previousBytes, { kind: "reasoning" }); - } - - feed(text: string): AdapterEvent[] { - if (!text) return []; - if (this.state === "streaming") return [{ type: "text_delta", text }]; - if (this.state === "thinking") { - this.replaceCarry("thinkingBuffer", this.thinkingBuffer + text); - return this.drainThinking(); - } - this.replaceCarry("preBuffer", this.preBuffer + text); - const stripped = this.preBuffer.trimStart(); - const openTag = OPEN_TAGS.find(tag => stripped.startsWith(tag)); - if (openTag) { - this.state = "thinking"; - this.closeTag = closeTagFor(openTag); - this.replaceCarry("thinkingBuffer", stripped.slice(openTag.length)); - this.replaceCarry("preBuffer", ""); - return this.drainThinking(); - } - if (stripped.length <= MAX_OPEN_TAG && isPossibleOpenTagPrefix(stripped)) return []; - this.state = "streaming"; - const out = this.preBuffer; - this.replaceCarry("preBuffer", ""); - return out ? [{ type: "text_delta", text: out }] : []; - } - - flush(): AdapterEvent[] { - if (this.state === "thinking") { - const out = this.thinkingBuffer; - this.replaceCarry("thinkingBuffer", ""); - this.state = "streaming"; - return out ? [{ type: "reasoning_raw_delta", text: out }] : []; - } - if (this.preBuffer) { - const out = this.preBuffer; - this.replaceCarry("preBuffer", ""); - this.state = "streaming"; - return [{ type: "text_delta", text: out }]; - } - return []; - } - - /** Release any partial tag/content carry when the owning stream stops early. */ - dispose(): void { - this.replaceCarry("preBuffer", ""); - this.replaceCarry("thinkingBuffer", ""); - this.closeTag = ""; - this.state = "streaming"; - } - - private drainThinking(): AdapterEvent[] { - const close = this.closeTag; - const idx = this.thinkingBuffer.indexOf(close); - if (idx >= 0) { - const thinking = this.thinkingBuffer.slice(0, idx); - const after = this.thinkingBuffer.slice(idx + close.length).trimStart(); - this.replaceCarry("thinkingBuffer", ""); - this.state = "streaming"; - const events: AdapterEvent[] = []; - if (thinking) events.push({ type: "reasoning_raw_delta", text: thinking }); - if (after) events.push({ type: "text_delta", text: after }); - return events; - } - if (this.thinkingBuffer.length <= MAX_CLOSE_TAG) return []; - // Never split a surrogate pair at the send boundary: a lone high - // surrogate at the end of one delta encodes as U+FFFD. Move the cut one - // unit earlier so the whole pair stays in the carry. - let cut = this.thinkingBuffer.length - MAX_CLOSE_TAG; - if (cut > 0 && cut < this.thinkingBuffer.length) { - const atCut = this.thinkingBuffer.charCodeAt(cut - 1); - if (atCut >= 0xd800 && atCut <= 0xdbff) cut -= 1; - } - const send = this.thinkingBuffer.slice(0, cut); - this.replaceCarry("thinkingBuffer", this.thinkingBuffer.slice(cut)); - return send ? [{ type: "reasoning_raw_delta", text: send }] : []; - } -} diff --git a/src/adapters/kiro/stream.ts b/src/adapters/kiro/stream.ts index d10ab1105c0..0536038e7dd 100644 --- a/src/adapters/kiro/stream.ts +++ b/src/adapters/kiro/stream.ts @@ -16,7 +16,7 @@ import { } from "../kiro-errors"; import { parseKiroEvent } from "../kiro-events"; import { noteKiroTransientThrottle } from "../kiro-retry"; -import { KiroThinkingParser } from "../kiro-thinking"; +import { InlineThinkTagParser } from "../inline-think-tags"; import { isCompleteKiroToolInput, kiroTruncationErrorMessage } from "../kiro-truncation"; import { isValidKiroConversationId } from "../kiro-wire"; import { tagKiroReasoningBlob } from "./reasoning"; @@ -319,7 +319,7 @@ async function* parseKiroAttemptEvents( let authoritativeUsage: OcxUsage | undefined; let stopReason: string | undefined; const fallbackEvents: AdapterEvent[] = []; - const thinking = new KiroThinkingParser(budget); + const thinking = new InlineThinkTagParser(budget); const retainedEventBytes = (event: AdapterEvent): number => Buffer.byteLength(JSON.stringify(event)); const retainEvent = (event: AdapterEvent): void => { diff --git a/src/adapters/openai-chat.ts b/src/adapters/openai-chat.ts index fa024e0cccb..e59096542ac 100644 --- a/src/adapters/openai-chat.ts +++ b/src/adapters/openai-chat.ts @@ -3,6 +3,7 @@ import { chatParallelToolCallsWireValue } from "./openai-chat/parallel-tool-call import type { AdapterRequest, IncomingMeta, ProviderAdapter } from "./base"; import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig, OcxUsage } from "../types"; import { modelInList } from "../types"; +import { createInlineThinkContentSplitter, splitInlineThinkContent } from "./inline-think-tags"; import { mapReasoningEffort, modelRecordValue } from "../reasoning-effort"; import { debugProviderDiagnostic } from "../lib/debug"; import { sseFieldValue } from "../lib/sse-decoder"; @@ -384,6 +385,12 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd // Gate on the routed model, not list length: a mixed openai-chat provider // can list MiniMax ids without putting every sibling on MiniMax semantics. const reasoningDetailsOptIn = modelInList(provider.reasoningDetailsModels, lastRequestedModelId ?? ""); + // A gateway with no server-side reasoning parser leaves thinking inline in `content` as + // blocks, which would otherwise render as the answer. Passthrough unless opted in. + const inlineThink = createInlineThinkContentSplitter(provider.inlineThinkTagModels, lastRequestedModelId, budget); + const emitContent = function* (events: AdapterEvent[]): Generator { + for (const event of events) { if (event.type === "text_delta") sawUserFacingOutput = true; yield event; } + }; const handleDataLine = function* (line: string): Generator { const rawPayload = sseFieldValue(line, "data"); @@ -391,6 +398,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd const payload = rawPayload.trim(); if (payload.length === 0) return "continue"; if (payload === "[DONE]") { + yield* emitContent(inlineThink.flush()); if ((yield* flushToolCalls()) === "terminate") return "terminate"; const stopReason = stopReasonFor(finishReason); yield { type: "done", usage: pendingUsage, ...(stopReason ? { stopReason } : {}) }; @@ -451,8 +459,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (reasoningText !== undefined) yield { type: "reasoning_raw_delta", text: reasoningText }; } if (typeof delta.content === "string" && delta.content.length > 0) { - sawUserFacingOutput = true; - yield { type: "text_delta", text: delta.content }; + yield* emitContent(inlineThink.feed(delta.content)); } const rawToolCalls = delta.tool_calls; @@ -591,6 +598,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } if (typeof choice.finish_reason === "string" && choice.finish_reason) { + yield* emitContent(inlineThink.flush()); if ((yield* flushToolCalls()) === "terminate") return "terminate"; } return "continue"; @@ -634,6 +642,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (buffer.length > 0) { if ((yield* handleDataLine(buffer)) === "terminate") return; } + yield* emitContent(inlineThink.flush()); const sawFinish = finishReason !== undefined; if (!sawFinish && pendingToolCalls.length > 0) { // Some OpenAI-compatible gateways close immediately after a complete function-call @@ -681,6 +690,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd } finally { budget.releaseRetained(bufferBytes, { kind: "live_transient" }); reasoningDetailTracker.release(); + inlineThink.dispose(); closeToolCalls(); reader.releaseLock(); } @@ -767,7 +777,7 @@ export function createOpenAIChatAdapter(provider: OcxProviderConfig): ProviderAd if (segments.length > 0) reasoningText = segments.map(s => s.text).join(""); } if (reasoningText !== undefined) events.push({ type: "reasoning_raw_delta", text: reasoningText }); - if (typeof msg.content === "string") events.push({ type: "text_delta", text: msg.content }); + if (typeof msg.content === "string") events.push(...splitInlineThinkContent(provider.inlineThinkTagModels, lastRequestedModelId, budget, msg.content)); const rawToolCalls = msg.tool_calls; if (rawToolCalls !== undefined && rawToolCalls !== null) { if (!Array.isArray(rawToolCalls)) { diff --git a/src/providers/derive.ts b/src/providers/derive.ts index 299b95cc413..cc60e657b8a 100644 --- a/src/providers/derive.ts +++ b/src/providers/derive.ts @@ -47,6 +47,7 @@ export interface DerivedKeyLoginProvider { requiresReasoningPlaceholderModels?: string[]; showThinkingSummary?: boolean; reasoningSplitModels?: string[]; + inlineThinkTagModels?: string[]; reasoningDetailsModels?: string[]; thinkingToggleModels?: string[]; thinkingBudgetModels?: string[]; @@ -286,6 +287,7 @@ export function providerConfigSeed(entry: ProviderRegistryEntry): OcxProviderCon ...(entry.requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels: [...entry.requiresReasoningPlaceholderModels] } : {}), ...(entry.showThinkingSummary !== undefined ? { showThinkingSummary: entry.showThinkingSummary } : {}), ...(entry.reasoningSplitModels ? { reasoningSplitModels: [...entry.reasoningSplitModels] } : {}), + ...(entry.inlineThinkTagModels ? { inlineThinkTagModels: [...entry.inlineThinkTagModels] } : {}), ...(entry.reasoningDetailsModels ? { reasoningDetailsModels: [...entry.reasoningDetailsModels] } : {}), ...(entry.thinkingToggleModels ? { thinkingToggleModels: [...entry.thinkingToggleModels] } : {}), ...(entry.thinkingBudgetModels ? { thinkingBudgetModels: [...entry.thinkingBudgetModels] } : {}), @@ -336,6 +338,7 @@ export function deriveKeyLoginMap(): Record { ...(entry.requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels: [...entry.requiresReasoningPlaceholderModels] } : {}), ...(entry.showThinkingSummary !== undefined ? { showThinkingSummary: entry.showThinkingSummary } : {}), ...(entry.reasoningSplitModels ? { reasoningSplitModels: [...entry.reasoningSplitModels] } : {}), + ...(entry.inlineThinkTagModels ? { inlineThinkTagModels: [...entry.inlineThinkTagModels] } : {}), ...(entry.reasoningDetailsModels ? { reasoningDetailsModels: [...entry.reasoningDetailsModels] } : {}), ...(entry.thinkingToggleModels ? { thinkingToggleModels: [...entry.thinkingToggleModels] } : {}), ...(entry.thinkingBudgetModels ? { thinkingBudgetModels: [...entry.thinkingBudgetModels] } : {}), @@ -602,6 +605,7 @@ export function enrichProviderFromRegistry(name: string, prov: OcxProviderConfig if (!prov.preserveReasoningContentModels && seed.preserveReasoningContentModels) prov.preserveReasoningContentModels = [...seed.preserveReasoningContentModels]; if (!prov.requiresReasoningPlaceholderModels && seed.requiresReasoningPlaceholderModels) prov.requiresReasoningPlaceholderModels = [...seed.requiresReasoningPlaceholderModels]; if (!prov.reasoningSplitModels && seed.reasoningSplitModels) prov.reasoningSplitModels = [...seed.reasoningSplitModels]; + if (!prov.inlineThinkTagModels && seed.inlineThinkTagModels) prov.inlineThinkTagModels = [...seed.inlineThinkTagModels]; if (!prov.reasoningDetailsModels && seed.reasoningDetailsModels) prov.reasoningDetailsModels = [...seed.reasoningDetailsModels]; if (!prov.thinkingToggleModels && seed.thinkingToggleModels) prov.thinkingToggleModels = [...seed.thinkingToggleModels]; if (!prov.thinkingBudgetModels && seed.thinkingBudgetModels) prov.thinkingBudgetModels = [...seed.thinkingBudgetModels]; diff --git a/src/providers/model-rename-migration.ts b/src/providers/model-rename-migration.ts index 0606fe3dcd9..ecc1242c4fd 100644 --- a/src/providers/model-rename-migration.ts +++ b/src/providers/model-rename-migration.ts @@ -113,6 +113,7 @@ const MODEL_ID_LISTS = [ "noPenaltyModels", "autoToolChoiceOnlyModels", "preserveReasoningContentModels", + "inlineThinkTagModels", "thinkingBudgetModels", "directReasoningEffortModels", ] as const; diff --git a/src/providers/registry/model-ids.ts b/src/providers/registry/model-ids.ts index 94de3fd7d99..4c02ca57cad 100644 --- a/src/providers/registry/model-ids.ts +++ b/src/providers/registry/model-ids.ts @@ -118,6 +118,7 @@ export const REGISTRY_FIELD_MODEL_ID_ROLES = { requiresReasoningPlaceholderModels: NONE, showThinkingSummary: NONE, reasoningSplitModels: NONE, + inlineThinkTagModels: NONE, reasoningDetailsModels: NONE, thinkingToggleModels: NONE, thinkingBudgetModels: NONE, diff --git a/src/providers/registry/types.ts b/src/providers/registry/types.ts index 0a31facef64..e04931bdb63 100644 --- a/src/providers/registry/types.ts +++ b/src/providers/registry/types.ts @@ -335,6 +335,8 @@ export interface ProviderRegistryEntry { */ showThinkingSummary?: boolean; reasoningSplitModels?: string[]; + /** See OcxProviderConfig.inlineThinkTagModels. */ + inlineThinkTagModels?: string[]; reasoningDetailsModels?: string[]; thinkingToggleModels?: string[]; thinkingBudgetModels?: string[]; @@ -358,6 +360,6 @@ export type ProviderConfigSeed = Pick< | "modelMaxInputTokens" | "defaultMaxOutputTokens" | "modelMaxOutputTokens" | "reasoningEfforts" | "modelReasoningEfforts" | "modelDefaultReasoningEfforts" | "reasoningEffortMap" | "modelReasoningEffortMap" | "reasoningWireFormat" | "noVisionModels" | "noReasoningModels" | "noTemperatureModels" | "noTopPModels" | "noPenaltyModels" - | "autoToolChoiceOnlyModels" | "preserveReasoningContentModels" | "requiresReasoningPlaceholderModels" | "reasoningSplitModels" | "reasoningDetailsModels" | "thinkingToggleModels" | "thinkingBudgetModels" | "escapeBuiltinToolNames" | "openaiChatEofTolerance" | "showThinkingSummary" + | "autoToolChoiceOnlyModels" | "preserveReasoningContentModels" | "requiresReasoningPlaceholderModels" | "reasoningSplitModels" | "inlineThinkTagModels" | "reasoningDetailsModels" | "thinkingToggleModels" | "thinkingBudgetModels" | "escapeBuiltinToolNames" | "openaiChatEofTolerance" | "showThinkingSummary" | "googleMode" | "project" | "location" | "headers" >; diff --git a/src/providers/resolved-model-policy.ts b/src/providers/resolved-model-policy.ts index 7c3553a3d26..3a018ee62da 100644 --- a/src/providers/resolved-model-policy.ts +++ b/src/providers/resolved-model-policy.ts @@ -37,7 +37,7 @@ export type StaticProviderPolicyField = | "supportsResponsesCustomTools" | "preserveResponsesReasoningContent" | "dropResponsesReasoningItems" | "modelSupportsReasoningSummaries" | "supportsVerbosity" | "modelSupportsVerbosity" | "responsesItemIdRepair" | "autoToolChoiceOnlyModels" - | "preserveReasoningContentModels" | "requiresReasoningPlaceholderModels" | "reasoningSplitModels" + | "preserveReasoningContentModels" | "requiresReasoningPlaceholderModels" | "reasoningSplitModels" | "inlineThinkTagModels" | "reasoningDetailsModels" | "thinkingToggleModels" | "thinkingBudgetModels" | "showThinkingSummary" | "escapeBuiltinToolNames" | "googleMode" | "project" | "location" | "modelCapabilities" | "modelAutoCompactTokenLimits" | "modelSuppressSyntheticMax" | "modelReasoningSummaryDelivery" @@ -228,7 +228,7 @@ export function resolveModelPolicy(input: ResolveModelPolicyInput): ResolvedMode "noVisionModels", "noReasoningModels", "noTemperatureModels", "noTopPModels", "noPenaltyModels", "noJsonSchemaModels", "autoToolChoiceOnlyModels", "preserveReasoningContentModels", "requiresReasoningPlaceholderModels", - "reasoningSplitModels", "reasoningDetailsModels", "thinkingToggleModels", "thinkingBudgetModels", + "reasoningSplitModels", "inlineThinkTagModels", "reasoningDetailsModels", "thinkingToggleModels", "thinkingBudgetModels", ] as const) putUnion(key, entry?.[key]); for (const directModel of entry?.directReasoningEffortModels ?? []) { const staleBudget = [directModel, ...(entry?.thinkingBudgetModels ?? [])]; diff --git a/src/router.ts b/src/router.ts index 84e4347bb61..7549de960fc 100644 --- a/src/router.ts +++ b/src/router.ts @@ -322,6 +322,7 @@ export function routedProviderConfig(providerName: string, provider: OcxProvider const preserveReasoningContentModels = staticPolicy.preserveReasoningContentModels; const requiresReasoningPlaceholderModels = staticPolicy.requiresReasoningPlaceholderModels; const reasoningSplitModels = staticPolicy.reasoningSplitModels; + const inlineThinkTagModels = staticPolicy.inlineThinkTagModels; const reasoningDetailsModels = staticPolicy.reasoningDetailsModels; const thinkingToggleModels = staticPolicy.thinkingToggleModels; const thinkingBudgetModels = staticPolicy.thinkingBudgetModels; @@ -464,6 +465,7 @@ export function routedProviderConfig(providerName: string, provider: OcxProvider ...(preserveReasoningContentModels ? { preserveReasoningContentModels } : {}), ...(requiresReasoningPlaceholderModels ? { requiresReasoningPlaceholderModels } : {}), ...(reasoningSplitModels ? { reasoningSplitModels } : {}), + ...(inlineThinkTagModels ? { inlineThinkTagModels } : {}), ...(reasoningDetailsModels ? { reasoningDetailsModels } : {}), ...(thinkingToggleModels ? { thinkingToggleModels } : {}), ...(thinkingBudgetModels ? { thinkingBudgetModels } : {}), diff --git a/src/server/auth-cors.ts b/src/server/auth-cors.ts index a0ef077dd5d..f5d71343199 100644 --- a/src/server/auth-cors.ts +++ b/src/server/auth-cors.ts @@ -1060,6 +1060,7 @@ const PROVIDER_CONFIG_FIELD_POLICY = { transientRetryOn5xx: "editor", retryOnReset: "editor", reasoningSplitModels: "editor", + inlineThinkTagModels: "editor", reasoningDetailsModels: "editor", thinkingToggleModels: "editor", thinkingBudgetModels: "editor", diff --git a/src/types/provider.ts b/src/types/provider.ts index 3dbf793d9fd..a36bf94074e 100644 --- a/src/types/provider.ts +++ b/src/types/provider.ts @@ -950,6 +950,16 @@ export interface OcxProviderConfig { * thinking separately in `reasoning_content` / `reasoning_details` instead of visible content. */ reasoningSplitModels?: string[]; + /** + * Model ids served by a gateway that runs no server-side reasoning parser, so a thinking model + * leaves its chain of thought inline in `content` as `` / `` / `` + * blocks and never sends `reasoning_content` or `reasoning_details`. Without this the whole + * chain of thought renders as the answer. The openai-chat adapter then splits those blocks back + * into reasoning. Off by default and narrow on purpose: 66 registry providers share this + * adapter, and a gateway that does parse reasoning must not have its visible content rewritten. + * Prefer a provider-side parser or `reasoningSplitModels` when the upstream supports either. + */ + inlineThinkTagModels?: string[]; /** * Model ids whose chat endpoint carries thinking as a structured `reasoning_details` array * (MiniMax M-series with `reasoning_split`): stream deltas repeat each detail's `text` as a diff --git a/structure/providers/chat-compat.md b/structure/providers/chat-compat.md index cf601215ed8..0ef28e53538 100644 --- a/structure/providers/chat-compat.md +++ b/structure/providers/chat-compat.md @@ -35,6 +35,24 @@ Shared parsing and streaming follow the [request-copy](../transports/byte-accoun ## Reasoning and tool-result compatibility +### Inline think-tag recovery + +A gateway that serves a thinking model without a server-side reasoning parser returns the chain +of thought inside `message.content` as `` / `` / `` blocks and sends +neither `reasoning_content` nor `reasoning_details`. `src/adapters/openai-chat.ts` recovers those +blocks into reasoning only for models listed in `inlineThinkTagModels`. The option is off by +default because 66 registry providers share this adapter and a gateway that does parse reasoning +must keep its visible content byte-exact. Once enabled the splitter still engages only for a +response that opens with a thinking tag, so an answer that merely mentions one is never rewritten; +after it engages it keeps splitting later blocks, because M-series models interleave thinking with +answer segments. A block left unterminated at end of stream flushes as reasoning rather than being +dropped. Whitespace between the leading block and the start of the answer is dropped as formatting +noise; once answer text has been emitted, whitespace after a later closing tag is preserved, +because a mid-answer block sits inside markdown or code where indentation is meaningful. +`src/adapters/inline-think-tags.ts` owns the parser and is shared with the Kiro adapter, +which consumes it in single-block mode. Regression coverage is in +`tests/adapters/openai/openai-chat-inline-think-tags.test.ts`. + Google tool-declaration narrowing is observed by the Google final compiler, not this shared Chat compatibility layer. Its endpoint profile and privacy boundary are specified in the [Google provider contract](google.md#google-tool-schema-loss-reporting). diff --git a/tests/adapters/openai/openai-chat-inline-think-tags.test.ts b/tests/adapters/openai/openai-chat-inline-think-tags.test.ts new file mode 100644 index 00000000000..c520c7b54bf --- /dev/null +++ b/tests/adapters/openai/openai-chat-inline-think-tags.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, test } from "bun:test"; +import { createOpenAIChatAdapter as createOpenAIChatAdapterProduction } from "../../../src/adapters/openai-chat"; +import type { AdapterEvent, OcxParsedRequest, OcxProviderConfig } from "../../../src/types"; +import { withTestTranslatorBudget } from "../../helpers/translator-budget"; + +const MODEL = "GLM-5.3-Flash"; + +function provider(optIn: boolean): OcxProviderConfig { + return { + adapter: "openai-chat", + baseUrl: "https://example.test/v1", + apiKey: "key", + ...(optIn ? { inlineThinkTagModels: [MODEL] } : {}), + }; +} + +function parsed(): OcxParsedRequest { + return { + modelId: MODEL, + stream: true, + options: {}, + context: { messages: [{ role: "user", content: "ping", timestamp: 0 }] }, + }; +} + +/** parseStream gates on the routed model, which buildRequest records. */ +function adapterFor(optIn: boolean) { + const adapter = withTestTranslatorBudget(createOpenAIChatAdapterProduction(provider(optIn))); + adapter.buildRequest(parsed()); + return adapter; +} + +function sse(...contents: string[]): Response { + const lines = contents.map(text => `data: ${JSON.stringify({ choices: [{ delta: { content: text } }] })}\n\n`); + lines.push('data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\n', "data: [DONE]\n\n"); + return new Response(lines.join("")); +} + +async function collect(gen: AsyncGenerator): Promise { + const out: AdapterEvent[] = []; + for await (const event of gen) if (event.type !== "heartbeat") out.push(event); + return out; +} + +function joined(events: AdapterEvent[], type: "text_delta" | "reasoning_raw_delta"): string { + return events + .filter((event): event is Extract => event.type === type) + .map(event => event.text) + .join(""); +} + +describe("openai-chat inline recovery", () => { + test("a gateway without a reasoning parser has its thinking split out of the answer", async () => { + const events = await collect(adapterFor(true).parseStream( + sse("weigh", "ing it up", "OCX_THINK_OK"), + )); + + expect(joined(events, "reasoning_raw_delta")).toBe("weighing it up"); + expect(joined(events, "text_delta")).toBe("OCX_THINK_OK"); + expect(events.at(-1)?.type).toBe("done"); + }); + + test("a tag split across chunk boundaries is still recognized", async () => { + const events = await collect(adapterFor(true).parseStream( + sse("whyanswer"), + )); + + expect(joined(events, "reasoning_raw_delta")).toBe("why"); + expect(joined(events, "text_delta")).toBe("answer"); + }); + + test("interleaved blocks keep every later thought out of the answer", async () => { + const events = await collect(adapterFor(true).parseStream( + sse("first", "part one ", "second", "part two"), + )); + + expect(joined(events, "reasoning_raw_delta")).toBe("firstsecond"); + expect(joined(events, "text_delta")).toBe("part one part two"); + }); + + test("a response that opens with ordinary text is never rewritten", async () => { + const answer = "A model may discuss a tag without thinking in it."; + const events = await collect(adapterFor(true).parseStream(sse(answer))); + + expect(joined(events, "text_delta")).toBe(answer); + expect(events.some(event => event.type === "reasoning_raw_delta")).toBe(false); + }); + + test("the blank line before the answer is dropped, but the answer's own indentation survives", async () => { + const events = await collect(adapterFor(true).parseStream( + sse("plan\n\nUse this:\n", "check", " indented line"), + )); + + expect(joined(events, "reasoning_raw_delta")).toBe("plancheck"); + expect(joined(events, "text_delta")).toBe("Use this:\n indented line"); + }); + + test("an unterminated block is flushed as reasoning rather than lost", async () => { + const events = await collect(adapterFor(true).parseStream( + sse("cut off mid thought"), + )); + + expect(joined(events, "reasoning_raw_delta")).toBe("cut off mid thought"); + expect(joined(events, "text_delta")).toBe(""); + }); + + test("without the opt-in the same stream stays byte-exact visible content", async () => { + const events = await collect(adapterFor(false).parseStream( + sse("weighing it up", "OCX_THINK_OK"), + )); + + expect(joined(events, "text_delta")).toBe("weighing it upOCX_THINK_OK"); + expect(events.some(event => event.type === "reasoning_raw_delta")).toBe(false); + }); + + test("the non-streaming path splits the same way", async () => { + const adapter = adapterFor(true); + const response = new Response(JSON.stringify({ + choices: [{ message: { role: "assistant", content: "quietlyOCX_THINK_OK" } }], + }), { headers: { "content-type": "application/json" } }); + + const events = await adapter.parseResponse(response); + + expect(joined(events, "reasoning_raw_delta")).toBe("quietly"); + expect(joined(events, "text_delta")).toBe("OCX_THINK_OK"); + }); +}); diff --git a/tests/fixtures/file-size-baseline.json b/tests/fixtures/file-size-baseline.json index 716b586605b..c77defa26eb 100644 --- a/tests/fixtures/file-size-baseline.json +++ b/tests/fixtures/file-size-baseline.json @@ -18,7 +18,7 @@ ".github/scripts/issue-quality.test.cjs": 2143, "gui/src/pages/Models.tsx": 2792, "gui/src/styles.css": 2958, - "src/adapters/openai-chat.ts": 822, + "src/adapters/openai-chat.ts": 827, "src/adapters/openai-responses.ts": 6, "src/bridge.ts": 7, "src/codex/auth-api.ts": 43, diff --git a/tests/fixtures/test-layout-expected.json b/tests/fixtures/test-layout-expected.json index d5413609fb8..38b6f25e9e7 100644 --- a/tests/fixtures/test-layout-expected.json +++ b/tests/fixtures/test-layout-expected.json @@ -898,6 +898,7 @@ "openai-chat-eof.test.ts": "adapters/openai", "openai-chat-hardening.test.ts": "adapters/openai", "openai-chat-image-normalization.test.ts": "adapters/openai", + "openai-chat-inline-think-tags.test.ts": "adapters/openai", "openai-chat-invalid-tool-call-diagnostics.test.ts": "adapters/openai", "openai-chat-model-suffix.test.ts": "adapters/openai", "openai-chat-native-policy.test.ts": "adapters/openai", diff --git a/tests/providers/kiro/kiro-stream.test.ts b/tests/providers/kiro/kiro-stream.test.ts index 336f264d021..5ad687ef6f7 100644 --- a/tests/providers/kiro/kiro-stream.test.ts +++ b/tests/providers/kiro/kiro-stream.test.ts @@ -2231,8 +2231,8 @@ describe("kiro adapter — non-streaming parseResponse", () => { describe("surrogate safety at kiro boundaries", () => { test("the reasoning carry never emits a delta ending on a lone high surrogate", async () => { - const { KiroThinkingParser } = await import("../../../src/adapters/kiro-thinking"); - const parser = new KiroThinkingParser(); + const { InlineThinkTagParser } = await import("../../../src/adapters/inline-think-tags"); + const parser = new InlineThinkTagParser(); // An astral char exactly at the carry/send boundary. const events = parser.feed("🎆aaaaaaaaaaa"); const emitted = JSON.stringify(events);