diff --git a/crates/jp_config/src/model/id.rs b/crates/jp_config/src/model/id.rs index e19500eb6..0171cefaa 100644 --- a/crates/jp_config/src/model/id.rs +++ b/crates/jp_config/src/model/id.rs @@ -595,6 +595,9 @@ pub enum ProviderId { /// Openrouter provider. /// See: . Openrouter, + /// vLLM provider: a self-hosted vLLM server with an OpenAI-compatible API. + /// See: . + Vllm, /// xAI provider. /// See: . /// UNIMPLEMENTED. @@ -619,6 +622,7 @@ impl ProviderId { Self::Ollama => "ollama", Self::Openai => "openai", Self::Openrouter => "openrouter", + Self::Vllm => "vllm", Self::Xai => "xai", Self::Test => "test", diff --git a/crates/jp_config/src/model/id_tests.rs b/crates/jp_config/src/model/id_tests.rs index c91aeac00..bba3c1ba9 100644 --- a/crates/jp_config/src/model/id_tests.rs +++ b/crates/jp_config/src/model/id_tests.rs @@ -42,6 +42,7 @@ fn the_variant_list_leaves_out_the_test_provider() { "ollama", "openai", "openrouter", + "vllm", "xai" ]); } diff --git a/crates/jp_config/src/providers/llm.rs b/crates/jp_config/src/providers/llm.rs index 6a834f760..ef92d4fc3 100644 --- a/crates/jp_config/src/providers/llm.rs +++ b/crates/jp_config/src/providers/llm.rs @@ -8,6 +8,7 @@ pub mod llamacpp; pub mod ollama; pub mod openai; pub mod openrouter; +pub mod vllm; use indexmap::IndexMap; use schematic::{Config, ConfigError}; @@ -27,6 +28,7 @@ use crate::{ ollama::{OllamaConfig, PartialOllamaConfig}, openai::{OpenaiConfig, PartialOpenaiConfig}, openrouter::{OpenrouterConfig, PartialOpenrouterConfig}, + vllm::{PartialVllmConfig, VllmConfig}, }, util::merge_nested_indexmap, validate::Validator, @@ -83,6 +85,10 @@ pub struct LlmProviderConfig { /// Openrouter API configuration. #[setting(nested)] pub openrouter: OpenrouterConfig, + + /// vLLM API configuration. + #[setting(nested)] + pub vllm: VllmConfig, } impl Validator for LlmProviderConfig { @@ -104,6 +110,7 @@ impl AssignKeyValue for PartialLlmProviderConfig { _ if kv.p("ollama") => self.ollama.assign(kv)?, _ if kv.p("openai") => self.openai.assign(kv)?, _ if kv.p("openrouter") => self.openrouter.assign(kv)?, + _ if kv.p("vllm") => self.vllm.assign(kv)?, _ => return missing_key(&kv), } @@ -126,6 +133,7 @@ impl PartialConfigDelta for PartialLlmProviderConfig { ollama: self.ollama.delta(next.ollama), openai: self.openai.delta(next.openai), openrouter: self.openrouter.delta(next.openrouter), + vllm: self.vllm.delta(next.vllm), } } @@ -148,6 +156,7 @@ impl PartialConfigDelta for PartialLlmProviderConfig { &path(prefix, "openrouter"), unsets, ), + vllm: self.vllm.delta(next.vllm), } } } @@ -164,6 +173,7 @@ impl FillDefaults for PartialLlmProviderConfig { ollama: self.ollama.fill_from(defaults.ollama), openai: self.openai.fill_from(defaults.openai), openrouter: self.openrouter.fill_from(defaults.openrouter), + vllm: self.vllm.fill_from(defaults.vllm), } } } @@ -184,6 +194,7 @@ impl ToPartial for LlmProviderConfig { ollama: self.ollama.to_partial(), openai: self.openai.to_partial(), openrouter: self.openrouter.to_partial(), + vllm: self.vllm.to_partial(), } } } diff --git a/crates/jp_config/src/providers/llm/vllm.rs b/crates/jp_config/src/providers/llm/vllm.rs new file mode 100644 index 000000000..ea307f815 --- /dev/null +++ b/crates/jp_config/src/providers/llm/vllm.rs @@ -0,0 +1,77 @@ +//! vLLM API configuration. +//! +//! A vLLM server speaks the OpenAI-compatible `/v1/chat/completions` dialect +//! and checks the Bearer token given to it with `--api-key`. +//! +//! ```toml +//! [providers.llm.vllm] +//! api_key_env = "VLLM_API_KEY" +//! base_url = "http://127.0.0.1:8000" +//! ``` + +use schematic::Config; + +use crate::{ + assignment::{AssignKeyValue, AssignResult, KvAssignment, missing_key}, + delta::{PartialConfigDelta, delta_opt}, + fill::FillDefaults, + partial::{ToPartial, partial_opt}, +}; + +/// vLLM API configuration. +#[derive(Debug, Clone, PartialEq, Config)] +#[config(rename_all = "snake_case")] +pub struct VllmConfig { + /// Environment variable that contains the API key. + #[setting(default = "VLLM_API_KEY")] + pub api_key_env: String, + + /// The base URL to use for API requests. + /// + /// The default is `http://127.0.0.1:8000`, which is the default URL for a + /// vLLM server. + #[setting(default = "http://127.0.0.1:8000")] + pub base_url: String, +} + +impl AssignKeyValue for PartialVllmConfig { + fn assign(&mut self, kv: KvAssignment) -> AssignResult { + match kv.key_string().as_str() { + "" => kv.try_merge_object(self)?, + "api_key_env" => self.api_key_env = kv.try_some_string()?, + "base_url" => self.base_url = kv.try_some_string()?, + _ => return missing_key(&kv), + } + + Ok(()) + } +} + +impl PartialConfigDelta for PartialVllmConfig { + fn delta(&self, next: Self) -> Self { + Self { + api_key_env: delta_opt(self.api_key_env.as_ref(), next.api_key_env), + base_url: delta_opt(self.base_url.as_ref(), next.base_url), + } + } +} + +impl FillDefaults for PartialVllmConfig { + fn fill_from(self, defaults: Self) -> Self { + Self { + api_key_env: self.api_key_env.or(defaults.api_key_env), + base_url: self.base_url.or(defaults.base_url), + } + } +} + +impl ToPartial for VllmConfig { + fn to_partial(&self) -> Self::Partial { + let defaults = Self::Partial::default(); + + Self::Partial { + api_key_env: partial_opt(&self.api_key_env, defaults.api_key_env), + base_url: partial_opt(&self.base_url, defaults.base_url), + } + } +} diff --git a/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap b/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap index fd285dbab..213438976 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__app_config_fields.snap @@ -54,6 +54,8 @@ expression: "AppConfig::fields()" "style.code.line_numbers", "providers.mcp", "providers.llm.aliases", + "providers.llm.vllm.api_key_env", + "providers.llm.vllm.base_url", "providers.llm.openrouter.api_key_env", "providers.llm.openrouter.app_name", "providers.llm.openrouter.app_referrer", diff --git a/crates/jp_config/src/snapshots/jp_config__tests__app_config_schema_shape.snap b/crates/jp_config/src/snapshots/jp_config__tests__app_config_schema_shape.snap index 7462967b0..8dffe8d0f 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__app_config_schema_shape.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__app_config_schema_shape.snap @@ -38,7 +38,7 @@ assistant: AssistantConfig id: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string parameters: ParametersConfig max_tokens: int | null @@ -136,7 +136,7 @@ conversation: ConversationConfig id: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string parameters: ParametersConfig max_tokens: int | null @@ -189,7 +189,7 @@ conversation: ConversationConfig id: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string parameters: ParametersConfig max_tokens: int | null @@ -256,7 +256,7 @@ conversation: ConversationConfig id: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string parameters: ParametersConfig max_tokens: int | null @@ -363,7 +363,7 @@ conversation: ConversationConfig id: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string parameters: ParametersConfig max_tokens: int | null @@ -584,7 +584,7 @@ conversation: ConversationConfig |: PartialModelIdOrAliasConfig |: PartialModelIdConfig name?: string | null - provider?: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" | null + provider?: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" | null |: string |: null parameters?: @@ -723,7 +723,7 @@ providers: ProviderConfig *: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string anthropic: AnthropicConfig api_key_env?: string @@ -753,6 +753,9 @@ providers: ProviderConfig app_name?: string app_referrer: string | null base_url?: string + vllm: VllmConfig + api_key_env?: string + base_url?: string mcp: *: McpProviderConfig |: StdioConfig @@ -800,7 +803,7 @@ style: StyleConfig id: ModelIdOrAliasConfig |: ModelIdConfig name: string - provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "xai" + provider: "anthropic" | "cerebras" | "deepseek" | "google" | "llamacpp" | "ollama" | "openai" | "openrouter" | "vllm" | "xai" |: string parameters: ParametersConfig max_tokens: int | null diff --git a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap index f4e8b5bb6..3e65999bf 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default.snap @@ -251,6 +251,10 @@ PartialAppConfig { app_referrer: None, base_url: None, }, + vllm: PartialVllmConfig { + api_key_env: None, + base_url: None, + }, }, mcp: {}, }, diff --git a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap index b94f00f54..bb828576b 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_default_values.snap @@ -517,6 +517,14 @@ Ok( "https://openrouter.ai", ), }, + vllm: PartialVllmConfig { + api_key_env: Some( + "VLLM_API_KEY", + ), + base_url: Some( + "http://127.0.0.1:8000", + ), + }, }, mcp: {}, }, diff --git a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap index fe52a9463..211e47ef1 100644 --- a/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap +++ b/crates/jp_config/src/snapshots/jp_config__tests__partial_app_config_empty_serialize.snap @@ -251,6 +251,10 @@ PartialAppConfig { app_referrer: None, base_url: None, }, + vllm: PartialVllmConfig { + api_key_env: None, + base_url: None, + }, }, mcp: {}, }, diff --git a/crates/jp_llm/src/provider.rs b/crates/jp_llm/src/provider.rs index caff8699f..9e51675b3 100644 --- a/crates/jp_llm/src/provider.rs +++ b/crates/jp_llm/src/provider.rs @@ -9,6 +9,7 @@ pub mod ollama; pub mod openai; pub(crate) mod openai_compat; pub mod openrouter; +pub mod vllm; use std::sync::atomic::{AtomicU64, Ordering}; @@ -24,6 +25,7 @@ use llamacpp::Llamacpp; use ollama::Ollama; use openai::Openai; use openrouter::Openrouter; +use vllm::Vllm; use crate::{ error::Result, model::ModelDetails, provider::mock::MockProvider, query::ChatQuery, @@ -56,6 +58,7 @@ pub fn get_provider(id: ProviderId, config: &LlmProviderConfig) -> Result Box::new(Ollama::try_from(&config.ollama)?), ProviderId::Openai => Box::new(Openai::try_from(&config.openai)?), ProviderId::Openrouter => Box::new(Openrouter::try_from(&config.openrouter)?), + ProviderId::Vllm => Box::new(Vllm::try_from(&config.vllm)?), ProviderId::Deepseek => todo!(), ProviderId::Xai => todo!(), @@ -113,6 +116,7 @@ pub(crate) fn build_request_value( ProviderId::Openrouter => { Openrouter::try_from(&config.openrouter)?.request_value(model, query) } + ProviderId::Vllm => Vllm::try_from(&config.vllm)?.request_value(model, query), ProviderId::Test | ProviderId::Deepseek | ProviderId::Xai => { unreachable!("{id:?} is not part of the request snapshot suite") } diff --git a/crates/jp_llm/src/provider/llamacpp.rs b/crates/jp_llm/src/provider/llamacpp.rs index db10c0545..5e8f17615 100644 --- a/crates/jp_llm/src/provider/llamacpp.rs +++ b/crates/jp_llm/src/provider/llamacpp.rs @@ -1,40 +1,29 @@ -use std::{mem, time::Duration}; +use std::time::Duration; use async_trait::async_trait; use base64::Engine as _; -use futures::{Stream, StreamExt as _, future, stream}; use jp_attachment::AttachmentContent; use jp_config::{ - assistant::tool_choice::ToolChoice, model::{ id::{ModelIdConfig, Name, ProviderId}, parameters::ReasoningConfig, }, providers::llm::llamacpp::LlamacppConfig, }; -use jp_conversation::{ - ConversationStream, - event::{ChatResponse, EventKind, ToolCallResponse}, - thread::text_attachments_to_xml, -}; -use reqwest_eventsource::{Event as SseEvent, EventSource, retry::Never}; +use jp_conversation::thread::text_attachments_to_xml; +use reqwest_eventsource::{EventSource, retry::Never}; use serde::Deserialize; use serde_json::{Value, json}; use tracing::{debug, trace, warn}; use super::{ EventStream, ModelDetails, - openai::parameters_with_strict_mode, - openai_compat::{merge_consecutive_assistant_messages, parse_chunk}, -}; -use crate::{ - error::{Error, StreamError}, - event::{Event, FinishReason}, - provider::Provider, - query::ChatQuery, - stream::{aggregator::reasoning::ReasoningExtractor, with_tool_call_keepalive}, - tool::ToolDefinition, + openai_compat::{ + assemble_event_stream, convert_events, convert_tool_choice, convert_tools, + to_system_messages, + }, }; +use crate::{error::Error, provider::Provider, query::ChatQuery, stream::with_tool_call_keepalive}; static PROVIDER: ProviderId = ProviderId::Llamacpp; @@ -114,306 +103,12 @@ impl Provider for Llamacpp { es.set_retry_policy(Box::new(Never)); Ok(with_tool_call_keepalive( - assemble_event_stream(es, is_structured), + assemble_event_stream(es, "llamacpp", is_structured), TOOL_CALL_KEEPALIVE_INTERVAL, )) } } -/// Assemble the provider-agnostic event stream from a raw SSE event source. -fn assemble_event_stream(events: S, is_structured: bool) -> EventStream -where - S: Stream> + Send + 'static, -{ - let mut state = StreamState { - extractor: ReasoningExtractor::default(), - tool_call_indices: Vec::new(), - reasoning_flushed: false, - message_flushed: false, - finished: false, - finish_reason: None, - is_structured, - }; - - let mut seen_error = false; - events - .take_while(move |event| { - // Include the first error before stopping: it must reach the - // handler below to be surfaced (or dropped once finished), and - // stopping prevents the EventSource from reconnecting after a - // terminal error. - let keep = !seen_error; - if event.is_err() { - seen_error = true; - } - future::ready(keep) - }) - .then(move |event| { - let result = handle_sse_event_sync(event, &mut state); - async move { - match result { - Ok(v) => stream::iter(v).boxed(), - Err(e) => { - stream::iter(vec![Err(StreamError::from_eventsource(e).await)]).boxed() - } - } - } - }) - .flatten() - .boxed() -} - -/// Mutable state carried across SSE events in a single stream. -struct StreamState { - extractor: ReasoningExtractor, - /// Tracks which tool call indices have been seen, so we can flush them on - /// finish. - tool_call_indices: Vec, - reasoning_flushed: bool, - /// Tracks whether `Event::flush(1)` (the message/structured index) has - /// already been emitted in this stream. - /// Without this gate, the `finish_reason` chunk and the `[DONE]` sentinel - /// both emit it, producing a spurious second flush that downstream - /// consumers can misinterpret as a re-dispatch signal. - message_flushed: bool, - /// Whether the terminal `Finished` event has been emitted. - /// Once set, a subsequent stream error is the benign connection close that - /// follows `[DONE]` and is dropped rather than surfaced to the retry layer. - finished: bool, - /// Captured from `finish_reason` in the last choice delta. - /// Emitted as `Event::Finished` when the `[DONE]` sentinel arrives. - finish_reason: Option, - is_structured: bool, -} - -type SseResult = std::result::Result>, reqwest_eventsource::Error>; - -/// Process a single SSE event into zero or more provider-agnostic events. -#[expect(clippy::too_many_lines)] -fn handle_sse_event_sync( - event: Result, - state: &mut StreamState, -) -> SseResult { - match event { - Ok(SseEvent::Open) => Ok(vec![]), - Ok(SseEvent::Message(msg)) => { - trace!(event = %msg.data, "Received event from Llamacpp API."); - - if msg.data == "[DONE]" { - // Finalize the reasoning extractor on stream end. - state.extractor.finalize(); - let mut events: Vec> = - drain_extractor(&mut state.extractor, state.is_structured) - .into_iter() - .map(Ok) - .collect(); - - // Flush reasoning if we never did. - if !state.reasoning_flushed { - events.push(Ok(Event::flush(0))); - state.reasoning_flushed = true; - } - - // Flush message content if we never did. - if !state.message_flushed { - events.push(Ok(Event::flush(1))); - state.message_flushed = true; - } - - // Drain any tool call indices that weren't flushed via - // `finish_reason`. In well-behaved streams this is empty — - // the safety net guards against a missing `finish_reason` - // chunk that would otherwise orphan the tool call buffer. - for index in state.tool_call_indices.drain(..) { - events.push(Ok(Event::flush(index))); - } - - events.push(Ok(Event::Finished( - state - .finish_reason - .take() - .unwrap_or(FinishReason::Completed), - ))); - state.finished = true; - return Ok(events); - } - - let Some(chunk) = parse_chunk(&msg.data, "llamacpp") else { - return Ok(vec![]); - }; - - let mut events = Vec::new(); - - for choice in &chunk.choices { - let delta = &choice.delta; - - // Reasoning via `reasoning_content` (deepseek / deepseek-legacy formats) - if let Some(reasoning) = &delta.reasoning_content - && !reasoning.is_empty() - { - events.push(Ok(Event::reasoning(0, reasoning.clone()))); - } - - // Content - // - // If reasoning_content was present, the server already - // separated reasoning from content (deepseek / - // deepseek-legacy). Otherwise, content may contain tags - // (none format) and needs the extractor. - if let Some(content) = &delta.content - && !content.is_empty() - { - // Server separated reasoning; content is pure text. - if delta.reasoning_content.is_some() { - flush_reasoning_if_needed(&mut events, &mut state.reasoning_flushed); - - if state.is_structured { - events.push(Ok(Event::structured(1, content.clone()))); - } else { - events.push(Ok(Event::message(1, content.clone()))); - } - } else { - // Might contain tags — feed through extractor. - state.extractor.handle(content); - events.extend( - drain_extractor(&mut state.extractor, state.is_structured) - .into_iter() - .map(Ok), - ); - } - } - - // Tool calls - if delta.tool_calls.is_some() { - // A tool call terminates this message's content. Release - // any extractor-held tail now, before the tool-call parts - // are emitted: downstream drains the in-progress markdown - // paragraph at the tool-call boundary, so a tail released - // afterwards would land in a fresh paragraph and render as - // a mid-word blank-line split. - state.extractor.finalize(); - events.extend( - drain_extractor(&mut state.extractor, state.is_structured) - .into_iter() - .map(Ok), - ); - } - - if let Some(tool_calls) = &delta.tool_calls { - flush_reasoning_if_needed(&mut events, &mut state.reasoning_flushed); - - for tc in tool_calls { - let index = tc.index as usize + 2; - - if !state.tool_call_indices.contains(&index) { - state.tool_call_indices.push(index); - } - - let id = tc.id.clone().unwrap_or_default(); - let name = tc - .function - .as_ref() - .and_then(|f| f.name.clone()) - .unwrap_or_default(); - if !id.is_empty() || !name.is_empty() { - events.push(Ok(Event::tool_call_start(index, id, name))); - } - - if let Some(args) = - tc.function.as_ref().and_then(|f| f.arguments.as_deref()) - { - events.push(Ok(Event::tool_call_args(index, args))); - } - } - } - - // Finish reason - if let Some(reason) = &choice.finish_reason { - state.extractor.finalize(); - events.extend( - drain_extractor(&mut state.extractor, state.is_structured) - .into_iter() - .map(Ok), - ); - - // Flush reasoning and message content before tool calls - // so they appear earlier in the conversation history. - if !state.reasoning_flushed { - events.push(Ok(Event::flush(0))); - state.reasoning_flushed = true; - } - if !state.message_flushed { - events.push(Ok(Event::flush(1))); - state.message_flushed = true; - } - - if matches!(reason.as_str(), "tool_calls" | "stop") { - for index in state.tool_call_indices.drain(..) { - events.push(Ok(Event::flush(index))); - } - } - - // Per the OpenAI spec. - match reason.as_str() { - "length" => { - // Active tool-call blocks are structurally - // incomplete when the model hits the token - // limit. Drop them here so the `[DONE]` safety - // net does not commit truncated arguments — - // mirrors `EventBuilder::drain` and the Google - // provider's MaxTokens behaviour. - state.tool_call_indices.clear(); - state.finish_reason = Some(FinishReason::MaxTokens); - } - "stop" => state.finish_reason = Some(FinishReason::Completed), - _ => {} - } - } - } - - Ok(events) - } - Err(e) => { - // A stream error after `Finished` is the benign close that - // follows `[DONE]`; drop it. Before completion it's a real - // transport failure (a dropped or stalled connection) that must - // surface so the retry layer can act on it. - if state.finished { Ok(vec![]) } else { Err(e) } - } - } -} - -/// Push a reasoning flush event if we haven't already. -fn flush_reasoning_if_needed(events: &mut Vec>, flushed: &mut bool) { - if !*flushed { - events.push(Ok(Event::flush(0))); - *flushed = true; - } -} - -/// Drain accumulated content from the `ReasoningExtractor` into events. -/// -/// Index convention matches Ollama: 0 = reasoning, 1 = message content. -fn drain_extractor(extractor: &mut ReasoningExtractor, is_structured: bool) -> Vec { - let mut events = Vec::new(); - - if !extractor.reasoning.is_empty() { - let reasoning = mem::take(&mut extractor.reasoning); - events.push(Event::reasoning(0, reasoning)); - } - - if !extractor.other.is_empty() { - let content = mem::take(&mut extractor.other); - if is_structured { - events.push(Event::structured(1, content)); - } else { - events.push(Event::message(1, content)); - } - } - - events -} - #[cfg(test)] impl Llamacpp { /// Build the llama.cpp wire request for `query` and serialize it to JSON @@ -561,97 +256,6 @@ fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(Value, bool Ok((body, is_structured)) } -/// Convert system prompt parts into a list of JSON message values. -fn to_system_messages(parts: Vec) -> impl Iterator { - parts - .into_iter() - .map(|content| json!({ "role": "system", "content": content })) -} - -/// Convert a conversation event stream into a list of JSON message values. -fn convert_events(events: ConversationStream) -> Vec { - let messages = events - .into_iter() - .filter_map(|event| match event.into_kind() { - EventKind::ChatRequest(request) => { - Some(json!({ "role": "user", "content": request.content })) - } - EventKind::ChatResponse(response) => match response { - ChatResponse::Message { message } => { - Some(json!({ "role": "assistant", "content": message })) - } - ChatResponse::Reasoning { reasoning } => { - // Use the `reasoning_content` field so the server can - // apply the correct template formatting. This avoids - // manually wrapping in `` tags. - Some(json!({ - "role": "assistant", - "reasoning_content": reasoning, - })) - } - ChatResponse::Structured { data } => { - Some(json!({ "role": "assistant", "content": data.to_string() })) - } - }, - EventKind::ToolCallRequest(request) => Some(json!({ - "role": "assistant", - "tool_calls": [{ - "id": request.id, - "type": "function", - "function": { - "name": request.name, - "arguments": Value::Object(request.arguments).to_string(), - }, - }], - })), - EventKind::ToolCallResponse(ToolCallResponse { id, result }) => Some(json!({ - "role": "tool", - "tool_call_id": id, - "content": match result { - Ok(content) | Err(content) => content, - }, - })), - _ => None, - }) - .collect(); - - merge_consecutive_assistant_messages(messages) -} - -/// Convert tool definitions to the OpenAI-compatible JSON format. -/// -/// If [`ToolChoice::Function`] is set, only include the named tool. llama.cpp -/// doesn't support calling a specific tool by name, but it supports `required` -/// mode, so we limit the tool list instead. -fn convert_tools(tools: Vec, tool_choice: &ToolChoice) -> Vec { - tools - .into_iter() - .map(|tool| { - json!({ - "type": "function", - "function": { - "name": tool.name, - "description": tool.docs.schema_description().unwrap_or_default(), - "parameters": parameters_with_strict_mode(&tool.parameters, true), - "strict": true, - }, - }) - }) - .filter(|tool| match tool_choice { - ToolChoice::Function(req) => tool["function"]["name"].as_str() == Some(req.as_str()), - _ => true, - }) - .collect() -} - -fn convert_tool_choice(choice: &ToolChoice) -> &str { - match choice { - ToolChoice::Auto => "auto", - ToolChoice::None => "none", - ToolChoice::Required | ToolChoice::Function(_) => "required", - } -} - impl Llamacpp { /// The context size the server was launched with, from `/props`. /// diff --git a/crates/jp_llm/src/provider/llamacpp_tests.rs b/crates/jp_llm/src/provider/llamacpp_tests.rs index 622fdf8b5..fcb316f3e 100644 --- a/crates/jp_llm/src/provider/llamacpp_tests.rs +++ b/crates/jp_llm/src/provider/llamacpp_tests.rs @@ -1,10 +1,7 @@ -use eventsource_stream::Event as MessageEvent; -use futures::StreamExt as _; -use jp_conversation::ConversationEvent; -use reqwest_eventsource::Error as SseError; +use jp_config::assistant::tool_choice::ToolChoice; use super::*; -use crate::{event::EventPart, provider::openai_compat::StreamChunk}; +use crate::provider::openai_compat::StreamChunk; fn qwen_model() -> LlamacppModel { serde_json::from_value(serde_json::json!({ @@ -121,66 +118,6 @@ fn create_request_asks_the_template_to_skip_thinking_when_reasoning_is_off() { ); } -fn sse_message(data: &str) -> SseEvent { - SseEvent::Message(MessageEvent { - data: data.to_owned(), - ..MessageEvent::default() - }) -} - -fn flush_indices(events: &[Result]) -> Vec { - events - .iter() - .filter_map(|e| match e { - Ok(Event::Flush { index, .. }) => Some(*index), - _ => None, - }) - .collect() -} - -#[test_log::test(tokio::test)] -async fn surfaces_stream_error_before_completion() { - // A transport error before `[DONE]` (a dropped or stalled connection) must - // surface as a `StreamError` so the retry layer can act on it, rather than - // being silently swallowed. - let content = sse_message( - r#"{"choices":[{"delta":{"content":"partial"},"index":0,"finish_reason":null}]}"#, - ); - let events = stream::iter(vec![Ok(content), Err(SseError::StreamEnded)]); - - let out: Vec<_> = assemble_event_stream(events, false).collect().await; - - assert!( - out.iter().any(std::result::Result::is_err), - "pre-completion stream error must surface, got {out:?}", - ); -} - -#[test_log::test(tokio::test)] -async fn swallows_stream_error_after_completion() { - // The connection close that follows `[DONE]` is the benign EOF; once the - // stream has emitted `Finished` it must not be surfaced as an error. - let content = - sse_message(r#"{"choices":[{"delta":{"content":"hi"},"index":0,"finish_reason":"stop"}]}"#); - let events = stream::iter(vec![ - Ok(content), - Ok(sse_message("[DONE]")), - Err(SseError::StreamEnded), - ]); - - let out: Vec<_> = assemble_event_stream(events, false).collect().await; - - assert!( - out.iter().all(std::result::Result::is_ok), - "post-completion close must not surface an error, got {out:?}", - ); - assert!( - matches!(out.last(), Some(Ok(Event::Finished(_)))), - "stream must end with Finished, got {:?}", - out.last(), - ); -} - #[test] fn parse_deepseek_format_reasoning_in_dedicated_field() { // The default `--reasoning-format deepseek`: reasoning arrives in @@ -307,268 +244,3 @@ fn parse_missing_optional_fields() { assert!(delta.tool_calls.is_none()); assert!(chunk.choices[0].finish_reason.is_none()); } - -#[test] -fn convert_events_merges_consecutive_tool_calls() { - use jp_conversation::event::ToolCallRequest; - - let mut events = ConversationStream::new_test(); - events.extend([ - ConversationEvent::now(ToolCallRequest { - id: "call_1".into(), - name: "tool_a".into(), - arguments: serde_json::Map::new(), - }), - ConversationEvent::now(ToolCallRequest { - id: "call_2".into(), - name: "tool_b".into(), - arguments: serde_json::Map::new(), - }), - ]); - - let messages = convert_events(events); - - // Should be merged into a single assistant message with 2 tool_calls. - assert_eq!(messages.len(), 1); - let tool_calls = messages[0]["tool_calls"].as_array().unwrap(); - assert_eq!(tool_calls.len(), 2); - assert_eq!(tool_calls[0]["function"]["name"], "tool_a"); - assert_eq!(tool_calls[1]["function"]["name"], "tool_b"); -} - -#[test] -fn convert_events_sends_reasoning_content_field() { - let mut events = ConversationStream::new_test(); - events.extend(std::iter::once(ConversationEvent::now( - ChatResponse::reasoning("step 1: think hard"), - ))); - - let messages = convert_events(events); - - assert_eq!(messages.len(), 1); - assert_eq!( - messages[0]["reasoning_content"].as_str().unwrap(), - "step 1: think hard" - ); -} - -#[test] -fn convert_events_merges_reasoning_and_message() { - let mut events = ConversationStream::new_test(); - events.extend([ - ConversationEvent::now(ChatResponse::reasoning("let me think...")), - ConversationEvent::now(ChatResponse::message("the answer is 42")), - ]); - - let messages = convert_events(events); - - // Reasoning + message should be merged into a single assistant message. - assert_eq!(messages.len(), 1); - assert_eq!( - messages[0]["reasoning_content"].as_str().unwrap(), - "let me think..." - ); - assert_eq!(messages[0]["content"].as_str().unwrap(), "the answer is 42"); -} - -#[test] -fn convert_tool_choice_values() { - assert_eq!(convert_tool_choice(&ToolChoice::Auto), "auto"); - assert_eq!(convert_tool_choice(&ToolChoice::None), "none"); - assert_eq!(convert_tool_choice(&ToolChoice::Required), "required"); - assert_eq!( - convert_tool_choice(&ToolChoice::Function("my_fn".into())), - "required" - ); -} - -/// `finish_reason: "length"` followed by `[DONE]` must not flush any pending -/// tool-call buffers. -/// When the model hits the token limit mid-tool-call, the arguments are -/// structurally incomplete; the safety-net drain on `[DONE]` would otherwise -/// commit them with truncated JSON (degraded to `{}`), which could re-dispatch -/// a partial call. -#[test] -fn length_finish_reason_drops_pending_tool_calls() { - let mut state = StreamState { - extractor: ReasoningExtractor::default(), - tool_call_indices: Vec::new(), - reasoning_flushed: false, - message_flushed: false, - finished: false, - finish_reason: None, - is_structured: false, - }; - - // Tool call delta with partial arguments. - let tool_chunk = r#"{ - "choices": [{ - "delta": { - "tool_calls": [{ - "index": 0, - "id": "call_abc", - "function": { "name": "run_me", "arguments": "{\"path\":" } - }] - }, - "index": 0, - "finish_reason": null - }] - }"#; - handle_sse_event_sync(Ok(sse_message(tool_chunk)), &mut state).unwrap(); - assert_eq!(state.tool_call_indices, vec![2]); - - // Terminal `"length"` chunk: should clear the pending tool-call index so - // the `[DONE]` safety net cannot commit the truncated buffer. - let finish_chunk = r#"{ - "choices": [{ - "delta": {}, - "index": 0, - "finish_reason": "length" - }] - }"#; - let finish_events = handle_sse_event_sync(Ok(sse_message(finish_chunk)), &mut state).unwrap(); - // Reasoning was already flushed when the tool-call chunk arrived, so only - // the message index flushes here. The tool-call index must NOT be in this - // list — that's the bug guard. - assert_eq!( - flush_indices(&finish_events), - vec![1], - "only message index should flush on length, got {finish_events:?}" - ); - assert!( - state.tool_call_indices.is_empty(), - "length must drop pending tool-call indices, got {:?}", - state.tool_call_indices, - ); - assert_eq!(state.finish_reason, Some(FinishReason::MaxTokens)); - - // `[DONE]` safety net: must NOT flush the tool-call index, and must - // finish with MaxTokens. - let done_events = handle_sse_event_sync(Ok(sse_message("[DONE]")), &mut state).unwrap(); - assert!( - flush_indices(&done_events).is_empty(), - "[DONE] after length must not flush any indices, got {done_events:?}" - ); - let last = done_events.last().unwrap().as_ref().unwrap(); - assert!( - matches!(last, Event::Finished(FinishReason::MaxTokens)), - "expected Finished(MaxTokens), got {last:?}" - ); -} - -/// A tool-call frame must release the extractor's held-back tail before -/// emitting any tool-call parts. -/// -/// The `ReasoningExtractor` withholds the last bytes of content (one less than -/// the `\n` opener) in case a tag is split across frames. -/// Downstream drains the in-progress markdown paragraph at the tool-call -/// boundary, so if the tail were released after `ToolCallPart::Start`, it would -/// land in a fresh paragraph and render as a mid-word blank-line split (e.g. -/// `…directo` then a blank line then `ries.`). -#[test] -fn tool_call_frame_releases_extractor_tail_before_tool_call_parts() { - let mut state = StreamState { - extractor: ReasoningExtractor::default(), - tool_call_indices: Vec::new(), - reasoning_flushed: false, - message_flushed: false, - finished: false, - finish_reason: None, - is_structured: false, - }; - - // A full paragraph in one frame, ending in a word long enough that the - // hold-back window splits it. - let content = - "Let me first check what tools are available to me for reading files and directories.\n\n"; - let content_chunk = serde_json::json!({ - "choices": [{ - "delta": { "content": content }, - "index": 0, - "finish_reason": null - }] - }); - let content_events = - handle_sse_event_sync(Ok(sse_message(&content_chunk.to_string())), &mut state).unwrap(); - - // The content frame withholds the tail while tag detection stays armed. - let content_emitted: String = content_events - .iter() - .filter_map(|e| match e.as_ref().ok() { - Some(Event::Part { - part: EventPart::Message(text), - .. - }) => Some(text.clone()), - _ => None, - }) - .collect(); - assert!( - !content_emitted.ends_with("directories.\n\n"), - "the tail should still be held back after the content frame: {content_emitted:?}" - ); - - // The tool-call frame releases the tail... - let tool_chunk = serde_json::json!({ - "choices": [{ - "delta": { - "tool_calls": [{ - "index": 0, - "id": "call_1", - "function": { "name": "describe_tools", "arguments": "{}" } - }] - }, - "index": 0, - "finish_reason": "tool_calls" - }] - }); - let tool_events = - handle_sse_event_sync(Ok(sse_message(&tool_chunk.to_string())), &mut state).unwrap(); - - let tail: String = tool_events - .iter() - .filter_map(|e| match e.as_ref().ok() { - Some(Event::Part { - part: EventPart::Message(text), - .. - }) => Some(text.clone()), - _ => None, - }) - .collect(); - assert_eq!( - format!("{content_emitted}{tail}"), - content, - "content must be preserved across the tool-call boundary" - ); - - // ...and it must precede every tool-call part in the emitted order, so the - // downstream paragraph drain at the tool-call boundary sees the complete - // paragraph. - let first_tool_call = tool_events - .iter() - .position(|e| { - matches!( - e.as_ref().ok(), - Some(Event::Part { - part: EventPart::ToolCall(_), - .. - }) - ) - }) - .unwrap(); - let last_message = tool_events - .iter() - .rposition(|e| { - matches!( - e.as_ref().ok(), - Some(Event::Part { - part: EventPart::Message(_), - .. - }) - ) - }) - .unwrap(); - assert!( - last_message < first_tool_call, - "the extractor tail must be emitted before the tool-call parts, got {tool_events:?}" - ); -} diff --git a/crates/jp_llm/src/provider/openai_compat.rs b/crates/jp_llm/src/provider/openai_compat.rs index 289963afc..33d193ef3 100644 --- a/crates/jp_llm/src/provider/openai_compat.rs +++ b/crates/jp_llm/src/provider/openai_compat.rs @@ -20,9 +20,26 @@ //! `StreamChoice::delta` is the one required field; a chunk whose choice omits //! it fails to parse, and both providers log a warning and skip that chunk. +use std::mem; + +use futures::{Stream, StreamExt as _, future, stream}; +use jp_config::assistant::tool_choice::ToolChoice; +use jp_conversation::{ + ConversationStream, + event::{ChatResponse, EventKind, ToolCallResponse}, +}; +use reqwest_eventsource::Event as SseEvent; use serde::Deserialize; use serde_json::{Value, json}; -use tracing::{debug, warn}; +use tracing::{debug, trace, warn}; + +use super::{EventStream, openai::parameters_with_strict_mode}; +use crate::{ + error::StreamError, + event::{Event, FinishReason}, + stream::aggregator::reasoning::ReasoningExtractor, + tool::ToolDefinition, +}; #[derive(Debug, Deserialize)] pub(crate) struct StreamChunk { @@ -169,6 +186,427 @@ pub(crate) fn merge_consecutive_assistant_messages(messages: Vec) -> Vec< }) } +/// Convert system prompt parts into a list of JSON message values. +pub(crate) fn to_system_messages(parts: Vec) -> impl Iterator { + parts + .into_iter() + .map(|content| json!({ "role": "system", "content": content })) +} + +/// Convert a conversation event stream into a list of JSON message values. +pub(crate) fn convert_events(events: ConversationStream) -> Vec { + let messages = events + .into_iter() + .filter_map(|event| match event.into_kind() { + EventKind::ChatRequest(request) => { + Some(json!({ "role": "user", "content": request.content })) + } + EventKind::ChatResponse(response) => match response { + ChatResponse::Message { message } => { + Some(json!({ "role": "assistant", "content": message })) + } + ChatResponse::Reasoning { reasoning } => { + // Use the `reasoning_content` field so the server can + // apply the correct template formatting. This avoids + // manually wrapping in `` tags. + Some(json!({ + "role": "assistant", + "reasoning_content": reasoning, + })) + } + ChatResponse::Structured { data } => { + Some(json!({ "role": "assistant", "content": data.to_string() })) + } + }, + EventKind::ToolCallRequest(request) => Some(json!({ + "role": "assistant", + "tool_calls": [{ + "id": request.id, + "type": "function", + "function": { + "name": request.name, + "arguments": Value::Object(request.arguments).to_string(), + }, + }], + })), + EventKind::ToolCallResponse(ToolCallResponse { id, result }) => Some(json!({ + "role": "tool", + "tool_call_id": id, + "content": match result { + Ok(content) | Err(content) => content, + }, + })), + _ => None, + }) + .collect(); + + merge_consecutive_assistant_messages(messages) +} + +/// Convert tool definitions to the OpenAI-compatible JSON format. +/// +/// If [`ToolChoice::Function`] is set, only include the named tool. +/// These servers don't support calling a specific tool by name, but they +/// support `required` mode, so we limit the tool list instead. +pub(crate) fn convert_tools(tools: Vec, tool_choice: &ToolChoice) -> Vec { + tools + .into_iter() + .map(|tool| { + json!({ + "type": "function", + "function": { + "name": tool.name, + "description": tool.docs.schema_description().unwrap_or_default(), + "parameters": parameters_with_strict_mode(&tool.parameters, true), + "strict": true, + }, + }) + }) + .filter(|tool| match tool_choice { + ToolChoice::Function(req) => tool["function"]["name"].as_str() == Some(req.as_str()), + _ => true, + }) + .collect() +} + +pub(crate) fn convert_tool_choice(choice: &ToolChoice) -> &'static str { + match choice { + ToolChoice::Auto => "auto", + ToolChoice::None => "none", + ToolChoice::Required | ToolChoice::Function(_) => "required", + } +} + +/// Assemble the provider-agnostic event stream from a raw SSE event source. +/// +/// `provider` names the server in log lines. +pub(crate) fn assemble_event_stream( + events: S, + provider: &'static str, + is_structured: bool, +) -> EventStream +where + S: Stream> + Send + 'static, +{ + let mut state = StreamState::new(provider, is_structured); + + let mut seen_error = false; + events + .take_while(move |event| { + // Include the first error before stopping: it must reach the + // handler below to be surfaced (or dropped once finished), and + // stopping prevents the EventSource from reconnecting after a + // terminal error. + let keep = !seen_error; + if event.is_err() { + seen_error = true; + } + future::ready(keep) + }) + .then(move |event| { + let result = handle_sse_event_sync(event, &mut state); + async move { + match result { + Ok(v) => stream::iter(v).boxed(), + Err(e) => { + stream::iter(vec![Err(StreamError::from_eventsource(e).await)]).boxed() + } + } + } + }) + .flatten() + .boxed() +} + +/// Mutable state carried across SSE events in a single stream. +pub(crate) struct StreamState { + /// The server name for log lines. + provider: &'static str, + extractor: ReasoningExtractor, + /// Tracks which tool call indices have been seen, so we can flush them on + /// finish. + pub(crate) tool_call_indices: Vec, + reasoning_flushed: bool, + /// Tracks whether `Event::flush(1)` (the message/structured index) has + /// already been emitted in this stream. + /// Without this gate, the `finish_reason` chunk and the `[DONE]` sentinel + /// both emit it, producing a spurious second flush that downstream + /// consumers can misinterpret as a re-dispatch signal. + message_flushed: bool, + /// Whether the terminal `Finished` event has been emitted. + /// Once set, a subsequent stream error is the benign connection close that + /// follows `[DONE]` and is dropped rather than surfaced to the retry layer. + finished: bool, + /// Captured from `finish_reason` in the last choice delta. + /// Emitted as `Event::Finished` when the `[DONE]` sentinel arrives. + pub(crate) finish_reason: Option, + /// Set when server-separated reasoning arrives, and cleared by the first + /// content frame that holds anything other than whitespace. + /// While set, leading whitespace is stripped from each content frame. + trim_content_prefix: bool, + is_structured: bool, +} + +impl StreamState { + pub(crate) fn new(provider: &'static str, is_structured: bool) -> Self { + Self { + provider, + extractor: ReasoningExtractor::default(), + tool_call_indices: Vec::new(), + reasoning_flushed: false, + message_flushed: false, + finished: false, + finish_reason: None, + trim_content_prefix: false, + is_structured, + } + } +} + +type SseResult = std::result::Result>, reqwest_eventsource::Error>; + +/// Process a single SSE event into zero or more provider-agnostic events. +#[expect(clippy::too_many_lines)] +pub(crate) fn handle_sse_event_sync( + event: Result, + state: &mut StreamState, +) -> SseResult { + match event { + Ok(SseEvent::Open) => Ok(vec![]), + Ok(SseEvent::Message(msg)) => { + trace!(provider = state.provider, event = %msg.data, "Received event."); + + if msg.data == "[DONE]" { + // Finalize the reasoning extractor on stream end. + state.extractor.finalize(); + let mut events: Vec> = + drain_extractor(&mut state.extractor, state.is_structured) + .into_iter() + .map(Ok) + .collect(); + + // Flush reasoning if we never did. + if !state.reasoning_flushed { + events.push(Ok(Event::flush(0))); + state.reasoning_flushed = true; + } + + // Flush message content if we never did. + if !state.message_flushed { + events.push(Ok(Event::flush(1))); + state.message_flushed = true; + } + + // Drain any tool call indices that weren't flushed via + // `finish_reason`. In well-behaved streams this is empty — + // the safety net guards against a missing `finish_reason` + // chunk that would otherwise orphan the tool call buffer. + for index in state.tool_call_indices.drain(..) { + events.push(Ok(Event::flush(index))); + } + + events.push(Ok(Event::Finished( + state + .finish_reason + .take() + .unwrap_or(FinishReason::Completed), + ))); + state.finished = true; + return Ok(events); + } + + let Some(chunk) = parse_chunk(&msg.data, state.provider) else { + return Ok(vec![]); + }; + + let mut events = Vec::new(); + + for choice in &chunk.choices { + let delta = &choice.delta; + + // Reasoning via `reasoning_content` (deepseek / deepseek-legacy formats) + if let Some(reasoning) = &delta.reasoning_content + && !reasoning.is_empty() + { + events.push(Ok(Event::reasoning(0, reasoning.clone()))); + state.trim_content_prefix = true; + } + + // Content + // + // If reasoning_content was present, the server already + // separated reasoning from content (deepseek / + // deepseek-legacy). Otherwise, content may contain tags + // (none format) and needs the extractor. + if let Some(content) = &delta.content + && !content.is_empty() + { + // A chat template puts a separator between the reasoning + // block and the answer. Some servers (vLLM) pass it through + // as content, others (llama.cpp) strip it before it reaches + // the wire; dropping it here spares the answer a pair of + // leading blank lines. + let content = if state.trim_content_prefix { + content.trim_start() + } else { + content.as_str() + }; + + if !content.is_empty() { + state.trim_content_prefix = false; + + // Server separated reasoning; content is pure text. + if delta.reasoning_content.is_some() { + flush_reasoning_if_needed(&mut events, &mut state.reasoning_flushed); + + if state.is_structured { + events.push(Ok(Event::structured(1, content.to_owned()))); + } else { + events.push(Ok(Event::message(1, content.to_owned()))); + } + } else { + // Might contain tags — feed through extractor. + state.extractor.handle(content); + events.extend( + drain_extractor(&mut state.extractor, state.is_structured) + .into_iter() + .map(Ok), + ); + } + } + } + + // Tool calls + if delta.tool_calls.is_some() { + // A tool call terminates this message's content. Release + // any extractor-held tail now, before the tool-call parts + // are emitted: downstream drains the in-progress markdown + // paragraph at the tool-call boundary, so a tail released + // afterwards would land in a fresh paragraph and render as + // a mid-word blank-line split. + state.extractor.finalize(); + events.extend( + drain_extractor(&mut state.extractor, state.is_structured) + .into_iter() + .map(Ok), + ); + } + + if let Some(tool_calls) = &delta.tool_calls { + flush_reasoning_if_needed(&mut events, &mut state.reasoning_flushed); + + for tc in tool_calls { + let index = tc.index as usize + 2; + + if !state.tool_call_indices.contains(&index) { + state.tool_call_indices.push(index); + } + + let id = tc.id.clone().unwrap_or_default(); + let name = tc + .function + .as_ref() + .and_then(|f| f.name.clone()) + .unwrap_or_default(); + if !id.is_empty() || !name.is_empty() { + events.push(Ok(Event::tool_call_start(index, id, name))); + } + + if let Some(args) = + tc.function.as_ref().and_then(|f| f.arguments.as_deref()) + { + events.push(Ok(Event::tool_call_args(index, args))); + } + } + } + + // Finish reason + if let Some(reason) = &choice.finish_reason { + state.extractor.finalize(); + events.extend( + drain_extractor(&mut state.extractor, state.is_structured) + .into_iter() + .map(Ok), + ); + + // Flush reasoning and message content before tool calls + // so they appear earlier in the conversation history. + if !state.reasoning_flushed { + events.push(Ok(Event::flush(0))); + state.reasoning_flushed = true; + } + if !state.message_flushed { + events.push(Ok(Event::flush(1))); + state.message_flushed = true; + } + + if matches!(reason.as_str(), "tool_calls" | "stop") { + for index in state.tool_call_indices.drain(..) { + events.push(Ok(Event::flush(index))); + } + } + + // Per the OpenAI spec. + match reason.as_str() { + "length" => { + // Active tool-call blocks are structurally + // incomplete when the model hits the token + // limit. Drop them here so the `[DONE]` safety + // net does not commit truncated arguments — + // mirrors `EventBuilder::drain` and the Google + // provider's MaxTokens behaviour. + state.tool_call_indices.clear(); + state.finish_reason = Some(FinishReason::MaxTokens); + } + "stop" => state.finish_reason = Some(FinishReason::Completed), + _ => {} + } + } + } + + Ok(events) + } + Err(e) => { + // A stream error after `Finished` is the benign close that + // follows `[DONE]`; drop it. Before completion it's a real + // transport failure (a dropped or stalled connection) that must + // surface so the retry layer can act on it. + if state.finished { Ok(vec![]) } else { Err(e) } + } + } +} + +/// Push a reasoning flush event if we haven't already. +fn flush_reasoning_if_needed(events: &mut Vec>, flushed: &mut bool) { + if !*flushed { + events.push(Ok(Event::flush(0))); + *flushed = true; + } +} + +/// Drain accumulated content from the `ReasoningExtractor` into events. +/// +/// Index convention matches Ollama: 0 = reasoning, 1 = message content. +fn drain_extractor(extractor: &mut ReasoningExtractor, is_structured: bool) -> Vec { + let mut events = Vec::new(); + + if !extractor.reasoning.is_empty() { + let reasoning = mem::take(&mut extractor.reasoning); + events.push(Event::reasoning(0, reasoning)); + } + + if !extractor.other.is_empty() { + let content = mem::take(&mut extractor.other); + if is_structured { + events.push(Event::structured(1, content)); + } else { + events.push(Event::message(1, content)); + } + } + + events +} + #[cfg(test)] #[path = "openai_compat_tests.rs"] mod tests; diff --git a/crates/jp_llm/src/provider/openai_compat_tests.rs b/crates/jp_llm/src/provider/openai_compat_tests.rs index 73e4b5389..51fc10e39 100644 --- a/crates/jp_llm/src/provider/openai_compat_tests.rs +++ b/crates/jp_llm/src/provider/openai_compat_tests.rs @@ -1,6 +1,413 @@ +use eventsource_stream::Event as MessageEvent; +use jp_conversation::{ConversationEvent, event::ToolCallRequest}; +use reqwest_eventsource::Error as SseError; use serde_json::json; use super::*; +use crate::event::EventPart; + +fn sse_message(data: &str) -> SseEvent { + SseEvent::Message(MessageEvent { + data: data.to_owned(), + ..MessageEvent::default() + }) +} + +fn flush_indices(events: &[Result]) -> Vec { + events + .iter() + .filter_map(|e| match e { + Ok(Event::Flush { index, .. }) => Some(*index), + _ => None, + }) + .collect() +} + +fn message_text(events: &[Result]) -> String { + events + .iter() + .filter_map(|e| match e.as_ref().ok() { + Some(Event::Part { + part: EventPart::Message(text), + .. + }) => Some(text.as_str()), + _ => None, + }) + .collect() +} + +/// Some servers (vLLM) pass the chat template's separator between the reasoning +/// block and the answer through as content, so the answer would otherwise open +/// with the template's blank lines. +#[test] +fn strips_the_template_separator_between_reasoning_and_content() { + let mut state = StreamState::new("test", false); + + let reasoning = json!({ + "choices": [{ + "delta": { "reasoning": "Deciding what to say.\n" }, + "index": 0, + "finish_reason": null + }] + }); + handle_sse_event_sync(Ok(sse_message(&reasoning.to_string())), &mut state).unwrap(); + + let content = json!({ + "choices": [{ + "delta": { "content": "\n\nTest received." }, + "index": 0, + "finish_reason": null + }] + }); + let mut events = + handle_sse_event_sync(Ok(sse_message(&content.to_string())), &mut state).unwrap(); + events.extend(handle_sse_event_sync(Ok(sse_message("[DONE]")), &mut state).unwrap()); + + assert_eq!(message_text(&events), "Test received."); +} + +/// The separator is only stripped where a reasoning block precedes the content. +/// Without one, leading blank lines are the model's own output and are kept. +#[test] +fn keeps_leading_blank_lines_when_no_reasoning_precedes_them() { + let mut state = StreamState::new("test", false); + + let content = json!({ + "choices": [{ + "delta": { "content": "\n\nTest received." }, + "index": 0, + "finish_reason": null + }] + }); + let mut events = + handle_sse_event_sync(Ok(sse_message(&content.to_string())), &mut state).unwrap(); + events.extend(handle_sse_event_sync(Ok(sse_message("[DONE]")), &mut state).unwrap()); + + assert_eq!(message_text(&events), "\n\nTest received."); +} + +/// A separator split across frames is stripped whole: the flag survives a frame +/// that turns out to be blank once trimmed. +#[test] +fn strips_a_separator_split_across_frames() { + let mut state = StreamState::new("test", false); + + let reasoning = json!({ + "choices": [{ + "delta": { "reasoning": "Deciding what to say.\n" }, + "index": 0, + "finish_reason": null + }] + }); + handle_sse_event_sync(Ok(sse_message(&reasoning.to_string())), &mut state).unwrap(); + + let mut events = vec![]; + for chunk in ["\n", "\n", "Test received."] { + let content = json!({ + "choices": [{ + "delta": { "content": chunk }, + "index": 0, + "finish_reason": null + }] + }); + events.extend( + handle_sse_event_sync(Ok(sse_message(&content.to_string())), &mut state).unwrap(), + ); + } + events.extend(handle_sse_event_sync(Ok(sse_message("[DONE]")), &mut state).unwrap()); + + assert_eq!(message_text(&events), "Test received."); +} + +#[test_log::test(tokio::test)] +async fn surfaces_stream_error_before_completion() { + // A transport error before `[DONE]` (a dropped or stalled connection) must + // surface as a `StreamError` so the retry layer can act on it, rather than + // being silently swallowed. + let content = sse_message( + r#"{"choices":[{"delta":{"content":"partial"},"index":0,"finish_reason":null}]}"#, + ); + let events = stream::iter(vec![Ok(content), Err(SseError::StreamEnded)]); + + let out: Vec<_> = assemble_event_stream(events, "test", false).collect().await; + + assert!( + out.iter().any(std::result::Result::is_err), + "pre-completion stream error must surface, got {out:?}", + ); +} + +#[test_log::test(tokio::test)] +async fn swallows_stream_error_after_completion() { + // The connection close that follows `[DONE]` is the benign EOF; once the + // stream has emitted `Finished` it must not be surfaced as an error. + let content = + sse_message(r#"{"choices":[{"delta":{"content":"hi"},"index":0,"finish_reason":"stop"}]}"#); + let events = stream::iter(vec![ + Ok(content), + Ok(sse_message("[DONE]")), + Err(SseError::StreamEnded), + ]); + + let out: Vec<_> = assemble_event_stream(events, "test", false).collect().await; + + assert!( + out.iter().all(std::result::Result::is_ok), + "post-completion close must not surface an error, got {out:?}", + ); + assert!( + matches!(out.last(), Some(Ok(Event::Finished(_)))), + "stream must end with Finished, got {:?}", + out.last(), + ); +} + +/// `finish_reason: "length"` followed by `[DONE]` must not flush any pending +/// tool-call buffers. +/// When the model hits the token limit mid-tool-call, the arguments are +/// structurally incomplete; the safety-net drain on `[DONE]` would otherwise +/// commit them with truncated JSON (degraded to `{}`), which could re-dispatch +/// a partial call. +#[test] +fn length_finish_reason_drops_pending_tool_calls() { + let mut state = StreamState::new("test", false); + + // Tool call delta with partial arguments. + let tool_chunk = r#"{ + "choices": [{ + "delta": { + "tool_calls": [{ + "index": 0, + "id": "call_abc", + "function": { "name": "run_me", "arguments": "{\"path\":" } + }] + }, + "index": 0, + "finish_reason": null + }] + }"#; + handle_sse_event_sync(Ok(sse_message(tool_chunk)), &mut state).unwrap(); + assert_eq!(state.tool_call_indices, vec![2]); + + // Terminal `"length"` chunk: should clear the pending tool-call index so + // the `[DONE]` safety net cannot commit the truncated buffer. + let finish_chunk = r#"{ + "choices": [{ + "delta": {}, + "index": 0, + "finish_reason": "length" + }] + }"#; + let finish_events = handle_sse_event_sync(Ok(sse_message(finish_chunk)), &mut state).unwrap(); + // Reasoning was already flushed when the tool-call chunk arrived, so only + // the message index flushes here. The tool-call index must NOT be in this + // list. + assert_eq!( + flush_indices(&finish_events), + vec![1], + "only message index should flush on length, got {finish_events:?}" + ); + assert!( + state.tool_call_indices.is_empty(), + "length must drop pending tool-call indices, got {:?}", + state.tool_call_indices, + ); + assert_eq!(state.finish_reason, Some(FinishReason::MaxTokens)); + + // `[DONE]` safety net: must NOT flush the tool-call index, and must + // finish with MaxTokens. + let done_events = handle_sse_event_sync(Ok(sse_message("[DONE]")), &mut state).unwrap(); + assert!( + flush_indices(&done_events).is_empty(), + "[DONE] after length must not flush any indices, got {done_events:?}" + ); + let last = done_events.last().unwrap().as_ref().unwrap(); + assert!( + matches!(last, Event::Finished(FinishReason::MaxTokens)), + "expected Finished(MaxTokens), got {last:?}" + ); +} + +/// A tool-call frame must release the extractor's held-back tail before +/// emitting any tool-call parts. +/// +/// The `ReasoningExtractor` withholds the last bytes of content (one less than +/// the `\n` opener) in case a tag is split across frames. +/// Downstream drains the in-progress markdown paragraph at the tool-call +/// boundary, so if the tail were released after `ToolCallPart::Start`, it would +/// land in a fresh paragraph and render as a mid-word blank-line split (e.g. +/// `…directo` then a blank line then `ries.`). +#[test] +fn tool_call_frame_releases_extractor_tail_before_tool_call_parts() { + let mut state = StreamState::new("test", false); + + // A full paragraph in one frame, ending in a word long enough that the + // hold-back window splits it. + let content = + "Let me first check what tools are available to me for reading files and directories.\n\n"; + let content_chunk = json!({ + "choices": [{ + "delta": { "content": content }, + "index": 0, + "finish_reason": null + }] + }); + let content_events = + handle_sse_event_sync(Ok(sse_message(&content_chunk.to_string())), &mut state).unwrap(); + + // The content frame withholds the tail while tag detection stays armed. + let content_emitted: String = content_events + .iter() + .filter_map(|e| match e.as_ref().ok() { + Some(Event::Part { + part: EventPart::Message(text), + .. + }) => Some(text.clone()), + _ => None, + }) + .collect(); + assert!( + !content_emitted.ends_with("directories.\n\n"), + "the tail should still be held back after the content frame: {content_emitted:?}" + ); + + // The tool-call frame releases the tail... + let tool_chunk = json!({ + "choices": [{ + "delta": { + "tool_calls": [{ + "index": 0, + "id": "call_1", + "function": { "name": "describe_tools", "arguments": "{}" } + }] + }, + "index": 0, + "finish_reason": "tool_calls" + }] + }); + let tool_events = + handle_sse_event_sync(Ok(sse_message(&tool_chunk.to_string())), &mut state).unwrap(); + + let tail: String = tool_events + .iter() + .filter_map(|e| match e.as_ref().ok() { + Some(Event::Part { + part: EventPart::Message(text), + .. + }) => Some(text.clone()), + _ => None, + }) + .collect(); + assert_eq!( + format!("{content_emitted}{tail}"), + content, + "content must be preserved across the tool-call boundary" + ); + + // ...and it must precede every tool-call part in the emitted order, so the + // downstream paragraph drain at the tool-call boundary sees the complete + // paragraph. + let first_tool_call = tool_events + .iter() + .position(|e| { + matches!( + e.as_ref().ok(), + Some(Event::Part { + part: EventPart::ToolCall(_), + .. + }) + ) + }) + .unwrap(); + let last_message = tool_events + .iter() + .rposition(|e| { + matches!( + e.as_ref().ok(), + Some(Event::Part { + part: EventPart::Message(_), + .. + }) + ) + }) + .unwrap(); + assert!( + last_message < first_tool_call, + "the extractor tail must be emitted before the tool-call parts, got {tool_events:?}" + ); +} + +#[test] +fn convert_events_merges_consecutive_tool_calls() { + let mut events = ConversationStream::new_test(); + events.extend([ + ConversationEvent::now(ToolCallRequest { + id: "call_1".into(), + name: "tool_a".into(), + arguments: serde_json::Map::new(), + }), + ConversationEvent::now(ToolCallRequest { + id: "call_2".into(), + name: "tool_b".into(), + arguments: serde_json::Map::new(), + }), + ]); + + let messages = convert_events(events); + + // Should be merged into a single assistant message with 2 tool_calls. + assert_eq!(messages.len(), 1); + let tool_calls = messages[0]["tool_calls"].as_array().unwrap(); + assert_eq!(tool_calls.len(), 2); + assert_eq!(tool_calls[0]["function"]["name"], "tool_a"); + assert_eq!(tool_calls[1]["function"]["name"], "tool_b"); +} + +#[test] +fn convert_events_sends_reasoning_content_field() { + let mut events = ConversationStream::new_test(); + events.extend(std::iter::once(ConversationEvent::now( + ChatResponse::reasoning("step 1: think hard"), + ))); + + let messages = convert_events(events); + + assert_eq!(messages.len(), 1); + assert_eq!( + messages[0]["reasoning_content"].as_str().unwrap(), + "step 1: think hard" + ); +} + +#[test] +fn convert_events_merges_reasoning_and_message() { + let mut events = ConversationStream::new_test(); + events.extend([ + ConversationEvent::now(ChatResponse::reasoning("let me think...")), + ConversationEvent::now(ChatResponse::message("the answer is 42")), + ]); + + let messages = convert_events(events); + + // Reasoning + message should be merged into a single assistant message. + assert_eq!(messages.len(), 1); + assert_eq!( + messages[0]["reasoning_content"].as_str().unwrap(), + "let me think..." + ); + assert_eq!(messages[0]["content"].as_str().unwrap(), "the answer is 42"); +} + +#[test] +fn convert_tool_choice_values() { + assert_eq!(convert_tool_choice(&ToolChoice::Auto), "auto"); + assert_eq!(convert_tool_choice(&ToolChoice::None), "none"); + assert_eq!(convert_tool_choice(&ToolChoice::Required), "required"); + assert_eq!( + convert_tool_choice(&ToolChoice::Function("my_fn".into())), + "required" + ); +} /// A chunk carrying content parses and is handed back. #[test] diff --git a/crates/jp_llm/src/provider/vllm.rs b/crates/jp_llm/src/provider/vllm.rs new file mode 100644 index 000000000..a1152fac0 --- /dev/null +++ b/crates/jp_llm/src/provider/vllm.rs @@ -0,0 +1,320 @@ +//! The vLLM provider: a self-hosted server that speaks the OpenAI-compatible +//! `/v1/chat/completions` dialect and checks a Bearer token. + +use std::{env, time::Duration}; + +use async_trait::async_trait; +use base64::Engine as _; +use jp_attachment::AttachmentContent; +use jp_config::{ + model::{ + id::{ModelIdConfig, Name, ProviderId}, + parameters::ReasoningConfig, + }, + providers::llm::vllm::VllmConfig, +}; +use jp_conversation::thread::text_attachments_to_xml; +use reqwest::header::{self, HeaderMap, HeaderValue}; +use reqwest_eventsource::{EventSource, retry::Never}; +use serde::Deserialize; +use serde_json::{Value, json}; +use tracing::{debug, trace, warn}; + +use super::{ + EventStream, ModelDetails, + openai_compat::{assemble_event_stream, convert_events, convert_tool_choice, convert_tools}, + trace_to_tmpfile, +}; +use crate::{error::Error, provider::Provider, query::ChatQuery, stream::with_tool_call_keepalive}; + +static PROVIDER: ProviderId = ProviderId::Vllm; + +/// How often to inject a synthetic keep-alive while a tool call is streaming. +/// +/// Stays below the enforced minimum `stream_idle_timeout_secs` (10s) so the +/// heartbeat always lands before the idle window elapses if the model pauses +/// between argument chunks. +const TOOL_CALL_KEEPALIVE_INTERVAL: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone)] +pub struct Vllm { + client: reqwest::Client, + base_url: String, +} + +#[async_trait] +impl Provider for Vllm { + async fn model_details(&self, name: &Name) -> Result { + let id: ModelIdConfig = (PROVIDER, name.as_ref()).try_into()?; + + Ok(self + .models() + .await? + .into_iter() + .find(|m| m.id == id) + .unwrap_or(ModelDetails::empty(id))) + } + + async fn models(&self) -> Result, Error> { + self.client + .get(format!("{}/v1/models", self.base_url)) + .send() + .await? + .error_for_status()? + .json::() + .await? + .data + .iter() + .map(map_model) + .collect::>() + } + + async fn chat_completion_stream( + &self, + model: &ModelDetails, + query: ChatQuery, + ) -> Result { + debug!(model = %model.id.name, "Starting vLLM chat completion stream."); + + let (body, is_structured) = create_request(model, query)?; + + trace!( + request = %trace_to_tmpfile("jp-vllm-request", &body), + "Request payload." + ); + + let request = self + .client + .post(format!("{}/v1/chat/completions", self.base_url)) + .header("content-type", "application/json") + .json(&body); + + let mut es = + EventSource::new(request).map_err(|e| Error::InvalidResponse(e.to_string()))?; + // Retries are owned by the stream retry layer; disable EventSource's + // own auto-reconnect so a closed connection ends the stream instead of + // silently re-issuing the request. + es.set_retry_policy(Box::new(Never)); + + Ok(with_tool_call_keepalive( + assemble_event_stream(es, "vllm", is_structured), + TOOL_CALL_KEEPALIVE_INTERVAL, + )) + } +} + +#[cfg(test)] +impl Vllm { + /// Build the vLLM wire request for `query` and serialize it to JSON without + /// sending. + /// Test-only seam for snapshotting request construction (notably compaction + /// projection) across providers. + #[expect( + clippy::unused_self, + reason = "uniform per-provider seam; only some providers read instance state" + )] + pub(crate) fn request_value( + &self, + model: &ModelDetails, + query: ChatQuery, + ) -> Result { + let (request, _) = create_request(model, query)?; + Ok(request) + } +} + +/// Build the JSON request body for the vLLM `/v1/chat/completions` endpoint. +/// +/// Returns `(body, is_structured)`. +fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(Value, bool), Error> { + let ChatQuery { + thread, + tools, + tool_choice, + } = query; + + let structured_schema = thread.events.schema(); + + let is_structured = structured_schema.is_some(); + let config = thread.events.config()?; + let parameters = &config.assistant.model.parameters; + let slug = model.id.name.to_string(); + + let parts = thread.into_parts(); + + let mut system_parts = parts.system_parts; + if let Some(xml) = text_attachments_to_xml(&parts.attachments)? { + system_parts.push(xml); + } + + // vLLM renders the request through the served model's own chat template, + // and several of those templates reject a system message that isn't the + // first message. Joining the parts keeps every served model reachable + // regardless of its template. + let mut messages: Vec = if system_parts.is_empty() { + vec![] + } else { + vec![json!({ "role": "system", "content": system_parts.join("\n\n") })] + }; + + // Prepend binary image attachments as a user message with image_url + // content blocks (OpenAI chat completions format). + let image_blocks: Vec<_> = parts + .attachments + .iter() + .filter_map(|a| match &a.content { + AttachmentContent::Binary { data, media_type } if media_type.starts_with("image/") => { + Some(json!({ + "type": "image_url", + "image_url": { + "url": format!( + "data:{media_type};base64,{}", + base64::engine::general_purpose::STANDARD.encode(data), + ), + }, + })) + } + AttachmentContent::Binary { media_type, .. } => { + warn!( + source = %a.source, + media_type, + "Unsupported binary attachment media type for vLLM, skipping." + ); + None + } + AttachmentContent::Text(_) => None, + }) + .collect(); + + if !image_blocks.is_empty() { + messages.push(json!({ + "role": "user", + "content": image_blocks, + })); + } + + messages.extend(convert_events(parts.events)); + let converted_tools = convert_tools(tools, &tool_choice); + let tool_choice_val = convert_tool_choice(&tool_choice); + + trace!( + slug, + messages_size = messages.len(), + tools_size = converted_tools.len(), + "Built vLLM request." + ); + + // Models such as Qwen3 default to thinking-on, so + // `chat_template_kwargs.enable_thinking` tells the chat template whether to + // prompt the model to think at all. Models whose template doesn't read the + // kwarg silently ignore it. + let reasoning_enabled = !matches!(parameters.reasoning, None | Some(ReasoningConfig::Off)); + + let mut body = json!({ + "model": slug, + "messages": messages, + "stream": true, + "chat_template_kwargs": { "enable_thinking": reasoning_enabled }, + }); + + if let Some(temperature) = parameters.temperature { + body["temperature"] = json!(temperature); + } + + if let Some(top_p) = parameters.top_p { + body["top_p"] = json!(top_p); + } + + if let Some(max_tokens) = parameters.max_tokens { + body["max_tokens"] = json!(max_tokens); + } + + if !converted_tools.is_empty() { + body["tools"] = json!(converted_tools); + body["tool_choice"] = json!(tool_choice_val); + } + + if let Some(schema) = structured_schema { + body["response_format"] = json!({ + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "schema": schema, + "strict": true, + }, + }); + } + + Ok((body, is_structured)) +} + +/// A `/v1/models` listing from vLLM. +/// +/// vLLM serves the OpenAI shape and adds `max_model_len` per entry, which is +/// the context window the server was launched with. +#[derive(Debug, Deserialize)] +struct VllmModelList { + #[serde(default)] + data: Vec, +} + +#[derive(Debug, Deserialize)] +struct VllmModel { + id: String, + + /// The served context window. + /// + /// Absent on servers that omit the vLLM extension fields. + #[serde(default)] + max_model_len: Option, +} + +/// Map a vLLM model listing entry to model details. +/// +/// The id keeps its full form, for example `Qwen/Qwen3-8B`, because vLLM +/// accepts only that form in a request. +fn map_model(model: &VllmModel) -> Result { + Ok(ModelDetails { + id: (PROVIDER, model.id.as_str()).try_into()?, + display_name: None, + context_window: model.max_model_len, + // vLLM reports no generation ceiling; it is bounded by the served + // context rather than a per-model limit. + max_output_tokens: None, + // Reasoning is a server-launch concern for vLLM, selected with + // `--reasoning-parser` rather than reported per model, so support stays + // unknown and an explicit request is passed through. + reasoning: None, + knowledge_cutoff: None, + deprecated: None, + structured_output: None, + prefill: None, + features: vec![], + }) +} + +impl TryFrom<&VllmConfig> for Vllm { + type Error = Error; + + fn try_from(config: &VllmConfig) -> Result { + let api_key = env::var(&config.api_key_env) + .map_err(|_| Error::MissingEnv(config.api_key_env.clone()))?; + + let client = reqwest::Client::builder() + .default_headers(HeaderMap::from_iter([( + header::AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {api_key}")) + .map_err(|_| Error::InvalidResponse("invalid API key".into()))?, + )])) + .build()?; + + Ok(Vllm { + client, + base_url: config.base_url.clone(), + }) + } +} + +#[cfg(test)] +#[path = "vllm_tests.rs"] +mod tests; diff --git a/crates/jp_llm/src/provider/vllm_tests.rs b/crates/jp_llm/src/provider/vllm_tests.rs new file mode 100644 index 000000000..dd0095aa0 --- /dev/null +++ b/crates/jp_llm/src/provider/vllm_tests.rs @@ -0,0 +1,240 @@ +use jp_config::{ + assistant::{sections::SectionConfig, tool_choice::ToolChoice}, + model::parameters::PartialReasoningConfig, +}; +use jp_conversation::{ + ConversationEvent, ConversationStream, + event::{ChatRequest, ChatResponse, ToolCallRequest, ToolCallResponse}, + thread::Thread, +}; +use serde_json::{Map, json}; + +use super::*; +use crate::tool::{ToolDefinition, ToolDocs}; + +fn qwen_model() -> VllmModel { + serde_json::from_value(json!({ + "id": "Qwen/Qwen3-8B", + "object": "model", + "owned_by": "vllm", + "max_model_len": 40_960, + })) + .unwrap() +} + +fn qwen_details() -> ModelDetails { + ModelDetails::empty((PROVIDER, "Qwen/Qwen3-8B").try_into().unwrap()) +} + +fn query(events: ConversationStream, tools: Vec) -> ChatQuery { + ChatQuery { + thread: Thread { + system_prompt: None, + sections: vec![], + attachments: vec![], + events, + }, + tools, + tool_choice: ToolChoice::Auto, + } +} + +/// vLLM reports the served context window as `max_model_len`, and the model id +/// keeps its vendor prefix because vLLM accepts only the full id. +#[test] +fn map_model_keeps_full_id_and_reads_max_model_len() { + let details = map_model(&qwen_model()).unwrap(); + + assert_eq!(details.id.name.as_ref(), "Qwen/Qwen3-8B"); + assert_eq!(details.context_window, Some(40_960)); + assert_eq!(details.reasoning, None); +} + +/// A plain message becomes one user message, with streaming on and thinking +/// off, because the test config has no reasoning setting. +#[test] +fn create_request_plain_message() { + let events = ConversationStream::new_test().with_turn("Hello"); + + let (body, is_structured) = create_request(&qwen_details(), query(events, vec![])).unwrap(); + + assert!(!is_structured); + assert_eq!( + body, + json!({ + "model": "Qwen/Qwen3-8B", + "messages": [{ "role": "user", "content": "Hello" }], + "stream": true, + "chat_template_kwargs": { "enable_thinking": false }, + }) + ); +} + +/// Regression: vLLM renders the request through the served model's own chat +/// template, and several of those templates reject a system message that isn't +/// the first message. +/// The prompt, its sections, and the attachment XML must therefore arrive as a +/// single system message. +#[test] +fn create_request_joins_system_parts_into_one_message() { + let query = ChatQuery { + thread: Thread { + system_prompt: Some("You are JP.".to_owned()), + sections: vec![ + SectionConfig::default().with_content("Rule 1."), + SectionConfig::default().with_content("Rule 2."), + ], + attachments: vec![], + events: ConversationStream::new_test().with_turn("test"), + }, + tools: vec![], + tool_choice: ToolChoice::Auto, + }; + + let (body, _) = create_request(&qwen_details(), query).unwrap(); + + assert_eq!( + body["messages"], + json!([ + { "role": "system", "content": "You are JP.\n\nRule 1.\n\nRule 2." }, + { "role": "user", "content": "test" }, + ]) + ); +} + +/// Whether the model thinks at all is the chat template's decision, driven by +/// `enable_thinking`. +#[test] +fn create_request_asks_the_template_to_think_when_reasoning_is_on() { + let mut events = ConversationStream::new_test().with_turn("Hello"); + let mut delta = jp_config::PartialAppConfig::empty(); + delta.assistant.model.parameters.reasoning = Some(PartialReasoningConfig::Auto); + events.add_config_delta(delta); + + let (body, _) = create_request(&qwen_details(), query(events, vec![])).unwrap(); + + assert_eq!( + body["chat_template_kwargs"], + json!({ "enable_thinking": true }) + ); + assert!(body.get("reasoning_format").is_none()); +} + +/// A tool call and its result become one assistant message with `tool_calls` +/// and one `tool` message, and the tool list uses the strict function shape. +#[test] +fn create_request_tool_call_round_trip() { + let mut events = ConversationStream::new_test().with_turn("Read the file"); + events.extend([ + ConversationEvent::now(ToolCallRequest { + id: "call_1".into(), + name: "read_file".into(), + arguments: serde_json::from_value(json!({ "path": "a.txt" })).unwrap(), + }), + ConversationEvent::now(ToolCallResponse { + id: "call_1".into(), + result: Ok("contents".into()), + }), + ]); + + let tool = ToolDefinition { + name: "read_file".into(), + docs: ToolDocs { + summary: Some("Read a file.".into()), + ..ToolDocs::default() + }, + parameters: json!({ + "type": "object", + "properties": { "path": { "type": "string" } }, + "required": ["path"], + }), + }; + + let (body, _) = create_request(&qwen_details(), query(events, vec![tool])).unwrap(); + + assert_eq!( + body["messages"], + json!([ + { "role": "user", "content": "Read the file" }, + { + "role": "assistant", + "tool_calls": [{ + "id": "call_1", + "type": "function", + "function": { "name": "read_file", "arguments": "{\"path\":\"a.txt\"}" }, + }], + }, + { "role": "tool", "tool_call_id": "call_1", "content": "contents" }, + ]) + ); + assert_eq!(body["tool_choice"], json!("auto")); + assert_eq!( + body["tools"], + json!([{ + "type": "function", + "function": { + "name": "read_file", + "description": "Read a file.", + "parameters": { + "type": "object", + "properties": { "path": { "type": "string" } }, + "required": ["path"], + "additionalProperties": false, + }, + "strict": true, + }, + }]) + ); +} + +/// A schema on the request becomes a strict `json_schema` response format. +#[test] +fn create_request_structured_schema() { + let schema: Map = serde_json::from_value(json!({ + "type": "object", + "properties": { "answer": { "type": "string" } }, + })) + .unwrap(); + let events = ConversationStream::new_test().with_turn(ChatRequest { + content: "Answer".into(), + schema: Some(schema), + author: None, + }); + + let (body, is_structured) = create_request(&qwen_details(), query(events, vec![])).unwrap(); + + assert!(is_structured); + assert_eq!( + body["response_format"], + json!({ + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "schema": { + "type": "object", + "properties": { "answer": { "type": "string" } }, + }, + "strict": true, + }, + }) + ); +} + +/// The prior assistant reply stays a plain message in the history. +#[test] +fn create_request_keeps_assistant_history() { + let mut events = ConversationStream::new_test().with_turn("Hi"); + events.extend([ConversationEvent::now(ChatResponse::message("Hello!"))]); + let events = events.with_turn("Again"); + + let (body, _) = create_request(&qwen_details(), query(events, vec![])).unwrap(); + + assert_eq!( + body["messages"], + json!([ + { "role": "user", "content": "Hi" }, + { "role": "assistant", "content": "Hello!" }, + { "role": "user", "content": "Again" }, + ]) + ); +} diff --git a/crates/jp_llm/src/provider_tests.rs b/crates/jp_llm/src/provider_tests.rs index c1532fa82..5542d219e 100644 --- a/crates/jp_llm/src/provider_tests.rs +++ b/crates/jp_llm/src/provider_tests.rs @@ -17,6 +17,7 @@ macro_rules! test_all_providers { mod openrouter{ use super::*; $(test_all_providers!(func; $fn, ProviderId::Openrouter);)* } mod ollama { use super::*; $(test_all_providers!(func; $fn, ProviderId::Ollama);)* } mod llamacpp { use super::*; $(test_all_providers!(func; $fn, ProviderId::Llamacpp);)* } + mod vllm { use super::*; $(test_all_providers!(func; $fn, ProviderId::Vllm);)* } }; (func; $fn:ident, $provider:ty) => { paste::paste! { diff --git a/crates/jp_llm/src/test.rs b/crates/jp_llm/src/test.rs index f83fbe8aa..f36d798dc 100644 --- a/crates/jp_llm/src/test.rs +++ b/crates/jp_llm/src/test.rs @@ -1,4 +1,4 @@ -use std::{panic, path::Path, sync::Arc}; +use std::{env, panic, path::Path, sync::Arc}; use chrono::{TimeZone as _, Utc}; use futures::TryStreamExt as _; @@ -458,14 +458,26 @@ pub async fn run_test( .await } -#[expect(clippy::too_many_lines)] -pub async fn run_chat_completion( - test_name: impl AsRef, - provider_id: ProviderId, - mut config: LlmProviderConfig, - requests: Vec, -) -> std::result::Result<(), Box> { - let vcr = Vcr::new(match provider_id { +/// Environment variable naming the server to record against. +const UPSTREAM_ENV: &str = "JP_TEST_UPSTREAM"; + +/// The server the recorder forwards to, which is the provider's configured base +/// URL unless `JP_TEST_UPSTREAM` names another one. +/// +/// A self-hosted provider is configured with a loopback address, which reaches +/// a server only on the machine doing the recording. +/// Naming a deployment in the environment records against that one instead. +/// It redirects the upstream alone: requests still travel through the mock +/// server that writes the cassette. +/// +/// Read on playback too, where it has no effect, because playback never +/// forwards. +fn record_upstream(provider_id: ProviderId, config: &LlmProviderConfig) -> String { + if let Ok(url) = env::var(UPSTREAM_ENV) { + return url; + } + + match provider_id { ProviderId::Anthropic => config.anthropic.base_url.clone(), ProviderId::Cerebras => config.cerebras.base_url.clone(), ProviderId::Google => config.google.base_url.clone(), @@ -473,9 +485,20 @@ pub async fn run_chat_completion( ProviderId::Ollama => config.ollama.base_url.clone(), ProviderId::Openai => config.openai.base_url.clone(), ProviderId::Openrouter => config.openrouter.base_url.clone(), + ProviderId::Vllm => config.vllm.base_url.clone(), _ => String::new(), - }) - .with_fixture_suffix(&provider_id.as_str()); + } +} + +#[expect(clippy::too_many_lines)] +pub async fn run_chat_completion( + test_name: impl AsRef, + provider_id: ProviderId, + mut config: LlmProviderConfig, + requests: Vec, +) -> std::result::Result<(), Box> { + let vcr = + Vcr::new(record_upstream(provider_id, &config)).with_fixture_suffix(&provider_id.as_str()); vcr.cassette( test_name.as_ref(), @@ -493,6 +516,7 @@ pub async fn run_chat_completion( ProviderId::Ollama => config.ollama.base_url = url, ProviderId::Openai => config.openai.base_url = url, ProviderId::Openrouter => config.openrouter.base_url = url, + ProviderId::Vllm => config.vllm.base_url = url, _ => {} } @@ -506,6 +530,7 @@ pub async fn run_chat_completion( ProviderId::Google => config.google.api_key_env = env, ProviderId::Openai => config.openai.api_key_env = env, ProviderId::Openrouter => config.openrouter.api_key_env = env, + ProviderId::Vllm => config.vllm.api_key_env = env, _ => {} } } @@ -769,6 +794,10 @@ pub(crate) fn fixture_attachment(path: impl AsRef) -> Attachment { Attachment::binary(path.as_ref().display().to_string(), data, media_type) } +#[expect( + clippy::too_many_lines, + reason = "one arm per provider; the table is flat by design" +)] pub(crate) fn test_model_details(id: ProviderId) -> ModelDetails { match id { ProviderId::Anthropic => ModelDetails { @@ -857,6 +886,18 @@ pub(crate) fn test_model_details(id: ProviderId) -> ModelDetails { prefill: None, features: vec![], }, + ProviderId::Vllm => ModelDetails { + id: "vllm/Qwen/Qwen3.8-Flash-Next-NVFP4".parse().unwrap(), + display_name: None, + context_window: Some(131_072), + max_output_tokens: None, + reasoning: None, + knowledge_cutoff: None, + deprecated: None, + structured_output: None, + prefill: None, + features: vec![], + }, ProviderId::Test => ModelDetails::empty("test/mock-model".parse().unwrap()), ProviderId::Xai => unimplemented!(), ProviderId::Deepseek => unimplemented!(), diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap index 02cfa1b45..53db4dd24 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap index abbe792a4..10b3c912f 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_fable_5_forced_tool_soft_forces__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap index b212d5bc5..464de1cbd 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_image_attachment__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap index 25ed49a0b..7ca0ba961 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap index a6c4e147d..22ec16f4d 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_adaptive_thinking__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap index 46035d578..a230fb769 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_opus_4_6_max_effort__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap index 6ae02dd08..69200fc15 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_redacted_thinking__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap index e83d646e5..fe47d7b8c 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_request_chaining__conversation_stream.snap @@ -215,6 +215,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap index cf595bbaa..45f711b4d 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap index fad5af547..5bae98a5d 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap index 2f4cee447..8f4a411e2 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap index ea573fa8c..53ce388b4 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap index 528e0ff14..c9e76c59e 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap index 4a6933967..580983dbd 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap index d661c6079..09333da10 100644 --- a/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/anthropic/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap index 984b905f0..6c504e464 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap index 00c729a77..7fb69c2c0 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap index 8a80fa4fb..34cea3ee4 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap index 8998f4085..7e0dec139 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap index cfb9ff551..653c3985c 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap index f353e1d03..3e9afb5be 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap index c551949bc..ab72c56e5 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap index c09a3150f..f4aac9940 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap index 97b0eec5e..ed6302d5d 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap index d63d8f9c6..143466b81 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_auto_omits_effort__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap index 851bd4c54..de165d026 100644 --- a/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/cerebras/test_unknown_model_off_sends_none__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap index 30075fc2a..f09f2fa8d 100644 --- a/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap index 50cb779e3..9660d4f03 100644 --- a/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_gemini_3_reasoning__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap index 5989403ee..3a4a1777a 100644 --- a/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_image_attachment__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap index 263e862d6..2b7c251f5 100644 --- a/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap index ce781587a..8dd367b5f 100644 --- a/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap index f10013c8e..78831f7c7 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap index 94ff60d15..61e7c7659 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap index 6f13d1f36..c9a211461 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap index 3b7638e2f..48abc5384 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap index c912b441f..5f86ca5f7 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap index 406e242d0..e355d40fd 100644 --- a/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap b/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap index 108727e96..aacc44f92 100644 --- a/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/google/test_unknown_model_inferred_thinking_level__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap index d68339ac8..be4170a28 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap index 30a93654a..79d49ae82 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_image_attachment__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap index 8086ed732..ca4ad1ce9 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap index 48abf484d..a7556eea0 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap index ddf37eaa1..1f1c5fc41 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap index 499818698..17916e0ea 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap index a455a4cd1..9f803e3db 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap index d985dfc24..a367fc471 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap index 812991126..ce5d2f16f 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap index 219e762a8..3b911d2e6 100644 --- a/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/llamacpp/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap index 9ac67ca17..f48756816 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap index ad9fc79f4..4744c7fe6 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_image_attachment__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap index 6d4b3d56e..a8daa1145 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap index dad24b32a..ef6542f53 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap index 5aa8c54b5..c6d023f00 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap index e49abe95b..6f03c0b17 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap index 1b700e129..669df3858 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap index e948cbd1e..105ce03a2 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap index aedad076a..c7c2812ae 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap index 5450fc417..1b0852326 100644 --- a/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/ollama/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap index 8d3121034..713f86885 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap index 9139eb7d7..ad85a8dc8 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_cache_off_sends_explicit_optout__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap index 6e11a4c19..110e56ce3 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_pro_reasoning_and_explicit_caching__conversation_stream.snap @@ -215,6 +215,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap index 6099e32e3..f4b68861e 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_gpt_5_6_prompt_cache_read_after_write__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap index e7c167ddc..5f28596e5 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_image_attachment__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap index 1e370d6c5..53cac156a 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap index ef2906b43..7272cdaa4 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_reasoning_history_replayed_to_reasoning_unsupported_model__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap index 60fcd7ec0..860ccecb3 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap index 9c87ef9b9..881119faa 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap index bdae0c9f5..c90a4bfbb 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap index 982d27ad1..3e02ccafa 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap index 03b0d6585..ee7934f0f 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap index e57be6444..6527cc0a4 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap index c677257a7..9323cd5d2 100644 --- a/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openai/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap index 7d08d1cda..ed17621a8 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/anthropic_test_sub_provider_event_metadata__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap index bba998777..39160f53e 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/google_test_sub_provider_event_metadata__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap index 70e71c438..8623ea407 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/minimax_test_sub_provider_event_metadata__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_anthropic_opus_5_parallel_tool_round_trip__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_anthropic_opus_5_parallel_tool_round_trip__conversation_stream.snap index 5828e4f6d..5404ad3db 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_anthropic_opus_5_parallel_tool_round_trip__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_anthropic_opus_5_parallel_tool_round_trip__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap index 37c971074..99341153e 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_chat_completion_stream__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap index ee4cb45d8..02317899b 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_image_attachment__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap index 354cc27d3..c81b01cdb 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_multi_turn_conversation__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap index 889fee0c8..58c27c393 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_structured_output__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap index e8ec1681a..1e6a691b6 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_auto__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap index d58498f40..f85621645 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_function__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap index e84297dca..353a13ea2 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap index 33d34571f..c45a12e60 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap index e5926a447..999fa982e 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_required_reasoning__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap index 30041cbf3..67ab9cba5 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/test_tool_call_stream__conversation_stream.snap @@ -210,6 +210,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap b/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap index 4eaebc373..012fbddca 100644 --- a/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap +++ b/crates/jp_llm/tests/fixtures/openrouter/x-ai_test_sub_provider_event_metadata__conversation_stream.snap @@ -213,6 +213,10 @@ expression: v "api_key_env": "OPENROUTER_API_KEY", "app_name": "JP", "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" } } }, diff --git a/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream.snap new file mode 100644 index 000000000..2a8ade826 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream.snap @@ -0,0 +1,33 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Reasoning { + reasoning: "We need respond to user \"Test message\". Simple. Need final. Maybe \"Test received. How can I help?\" in English.\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "Test received. I’m here and ready to help.", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream.yml b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream.yml new file mode 100644 index 000000000..8c668fb81 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream.yml @@ -0,0 +1,64 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Test message" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": true + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"We"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" need respond to user"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"Test message\". Simple"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". Need final."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Maybe"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"Test received"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" How can I help"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"?\" in"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" English.\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n\nTest received."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"’m"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" here and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" ready to help."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-86b2a1c3a652db32","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream__conversation_stream.snap new file mode 100644 index 000000000..afee5d9fb --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream__conversation_stream.snap @@ -0,0 +1,245 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": { + "effort": "low", + "exclude": false + }, + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Test message" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "reasoning": "We need respond to user \"Test message\". Simple. Need final. Maybe \"Test received. How can I help?\" in English.\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "Test received. I’m here and ready to help." + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream__raw_events.snap new file mode 100644 index 000000000..ec8189c4e --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_chat_completion_stream__raw_events.snap @@ -0,0 +1,145 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 0, + part: Reasoning( + "We", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " need respond to user", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "Test message\". Simple", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". Need final.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Maybe", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "Test received", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ".", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " How can I help", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "?\" in", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " English.\n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "Test re", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ce", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ived", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + ". I’m h", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ere and ready t", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "o help.", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_image_attachment.snap b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment.snap new file mode 100644 index 000000000..b60593c93 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment.snap @@ -0,0 +1,22 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "apple", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_image_attachment.yml b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment.yml new file mode 100644 index 000000000..1352d1c84 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment.yml @@ -0,0 +1,43 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQABAAD/2wCEAAkGBwgHBgkIBwgKCgkLDRYPDQwMDRsUFRAWIB0iIiAdHx8kKDQsJCYxJx8fLT0tMTU3Ojo6Iys/RD84QzQ5OjcBCgoKDQwNGg8PGjclHyU3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3Nzc3N//AABEIAJQAlAMBIgACEQEDEQH/xAAcAAEAAQUBAQAAAAAAAAAAAAAABAECAwUHBgj/xAA+EAACAgECBAMFBgIHCQAAAAABAgADEQQhBRIxQQYTUSJhcYGRBzKhscHRQnIUI1JTYrLwFRYkMzSSotLh/8QAGgEBAAIDAQAAAAAAAAAAAAAAAAIDAQQFBv/EACoRAAMAAgEDAwMDBQAAAAAAAAABAgMRBBIhMQUTQRQiMhVxoSQzQlFh/9oADAMBAAIRAxEAPwDuMREAREQBERAEREARLHbkQsSAB1J7TW8J8QcM4xqL6OHapLnoVWsCjoDnH5TG14M6fk2sQImTAiIgCIiAIiIAiIgCIiAIiIAlJZbYlVZssdURRksxwBPJcW+0HhOjLV6Ln11o/udqwfex/TMw6U+S3FhyZXqFs9hmMzk+p8c8d1hPlPp9LWf7mvJHxZifyEncOs13E1zfxbWh/RbmUH5Ayl55+Df/AEnLM9WRpfydF1NNWposovrWymxSjowyGUjBBnOvs00C8O8VceoVQvlItW3flssGfwEmpw+9bFVuI6sk7c3nv3+c0nD9PqqfFXE69JrbqnVAWsDEl846569+sqvJ98vRhcNJOVfk6xmVnhaeJeI9K2RbTqa1O/nJuR7iMfrJun8caXnarWaTUV2r1NaeYv1l85oZRXBzL8Vv9j1mZWaTTeKOF3rzG5qR0zdWUH1mz0us02rXm02oquH+BwfyliaZrVjufyWiRERMkBERAEREAREQBIHF+J08L0putV3Y7V1IMtY3YASXfalFL22EKijJJnHfHHjBtZrbNDorOUN7FtqndV7ovp3yfl8IZLULbNvh8W+TkUrwQ/EviPX8Y1TVaqwMEbbS1Niqr+Y/xGQ9DoRqQBa7Hl3OGwo+Ui6DTqawFQYPTPf3/GegpVKKwFXJPqe/r+InO9zqez2U4ceCFEIwnhpqHPVuoHTJH0my4TrbtO2BpbLO2QwEwULdqWJAbA2we09Lw3S+TWvNjCj6ySxdfk1OTUzOq7kY8S1JPN/QLvdyspMi6PSa48S1uuVRUdTgYcZKgZ/eehJUegEo2pqVcbZkvppRzupf4yQF4ffY3/EX22fFtvpMVvCKyjewAR3l2t4wKnAWWvxZHpyRjsczLmPBdMZuz0aTWcMrBIxj1wJrrkfS2C3S22U2jo9bcp+s2Go1Lq+PaOfaUj0mvt1ILFbW6jKZ/Kat1012OjMNz9x7Hwt41s81NDxwrk+ymq6ZPo37z3ykMAVIIO+ROB3cp9gAkHoJ7T7O/E7pq14HxCzKuM6Nyc42/wCXn4DI+Y9JucfkdX2s4XqPp6lPJiX7o6VEpKzdOIIiIAiJSAc6+13xO3CdBTw3R28ur1J5mI6qonIdCtgYY6kg5O82XjniJ41414jarc1dN3k1nthdv0kagIoXAweX3dZzOTk3Wj2HpGBY8Sfyzf6FzWQCSzEbZmypurdsnJY9TNBo8lPNJx2rz3mVTcg5l5mHbbrvNaa0zqVKbPVUa9KUxWvTqZSzjWoDBVYAE9czS6e4rQpfmUsMlZg1Go52JBOe+RJ++0ULjw67rZs7uM6i1yS5AXOMd5F1HFr+QnmI3xj1M1fOzEsuNh0PQTBqXsrNaNvnBGe49ZD6ht6NmMONdtG3TW23IGtOzbcx9R2kqjUh08th8JoEsVsNy8p2DZkrSvy2IVG246yXXtEblJG4sYEhhuyma3WAtW1iqDysD/8AJke5lDYIOxOTtiYiwNZORytvj4yO9lCejBYxdVbJ5Oq9yJFvaygpqKbPLupcOjf2WU5B+ozMxcYwOnYSLrGJGVUjGxErx01RVkW1o774Z4ovGuA6LiKgKb6wXUHPK/Rh8jmbWc1+xTXl+F8R4cx/6a8WIvorj91P1M6UOk70V1SmeN5GP28tT/oRESRSJa5wjEdhLpQjII9YB8rrzPrdUx38252+rGbRKAPZckFV6AZPWQzSaOJayo7FNTYPhhyJPpcjUbHO2fiJxM23Z7jiPWNaJiOyVFG/h+8AO8m6e/FasxBJGy/Ca2xyzWHOwABHTt1l2l1ShMkZA6H1EqTaZsVLZMvuDg55QQOuN5q3u5RzgE5bodj19Ir1BsezoFLHl+GZg093mNy8uWBJJPTEg/Oy6J0jIuoBbOMe7MGxmIPUgcvL6e6WuMb4+hlic6DDoOXOck7zGkS2ZaHKl8n4SaxxVWykFWIPpiQUsD5QsAo3AJ659JLquD1BegG2ZNdiu9szV2FwwAOQMY/SWFyECMDkby2sFS7E9TnrL61V2DYOff3k0UeDDYpCEBsHc4kS48yqN9xv7jJ2qKuNjg5+70kF8mp+Y5ct9785BfkRfjZ7P7GLmTxJxGnO1mjDfNWH/uZ2Sca+x8E+LtUQDgaF/wDOk7LO1xv7aPJeor+oYiIl5oiUMrKQD518S6Q6LxjxegjAGsawAjqGPOP834SzTIGtDbd+nQT1v2t8LbT+ItPxJAfL1dHlk/40z+hH0nnk5a8FE67HPpORmhrIz13AzdWCWYrqlNo5BsV65mOrTA5BPKMgZx0/1tJipzKrMQMEgYlSKymAGyp7d5DWzfVs02nrarWMg5ioz8xLFApodQnt2bqR2E2dykajmA+8MfhKpUowCqlW2BxIuC7rNXWjuV5Qd98+sksoAJsI3IHJ3Ik3yQPYQjGc8uM5Pxlt+nU8pXG7DBlbkde2a86MLYGsdQgOBn1krRBUDkrnP3eaSnpDVtz/AHs+yAOktZFwo3zmSlEarfkqteafZ+6MjmPvllSlVVFH3FySfwkxKtimBhdwD3OZkrpUN7LYLbEYkkip0a21XN/mKmcLgemZhFXIPvBjnebeylSCOwPaa+9fLscjlC8pLSFT37FdXtHtPsb0jnXcU1zdPKrqX3EksR9OWdTnk/s14cND4YpsIw+rY3k+oOy/+IH1nrJ2sM9MJHkOXk9zPTEREtNYREQDz3jng/8Atjw9dUi819JF9P8AMvb5jI+c5XbXyAjGFPUTujbicx8ZcHOh4k1lQxVd7SntnO4mtnx77nV9N5PQ/bZ5JkHUZAGDKcqli1eFXqFJ6zK/Mte+6g7j3S0ZDIU77742xNDWj0cVtbMTctgAK/1i98yoYvXvkqp6YlzAYYEEHY/OWixQoLAbnoczBcOYKHZcFh2EyafNlAZuoJwPdLdMpVLPZHtn8JmRwWbA69j2kWRbLXTCqxIBznGcyoQGxQ7ZBG+BK4QnlUktjMzOBy5IJyu2I2RdaKMoNtYyAuDv8JfhgyuHIO2QB0zLjlDzDHKvQzGWXOCTt1I3kkVtmblQBRkY5frKcO4E3FuMU6QD2G9qwj+EdzKUb2KObPfadJ8IcKGk051dq/1twAHuXtL8OLrfc5vN5Hsw9eWb+ipKakqrUKiKFUDsAJkiJ0jzYiIgCIiAJruOcMr4poH07kButbf2WmxiYa2tGZbl7RxHiOlt0uqem4FXRuV0Mhux5MIPZyem+PlOq+LfDa8Yo87T8qaxBgE9HHof0M5Prlu0Woeq+p6rkOGVlwROZnxuHv4PS8LmTkn/AKWlya/awSdm/aKythKsCG7AzALFK5ZiNhKJchby3OT1GG6ma/WdSci0S05xWcMEPQjMupfGCxAw2fl+sh1Xqpdcb5yM9pf53I5ZsHOctMdQdbJjkC4sSTkeneZFsc9yT2zsBIVNgtHNkD+LrK36ojKDoB3kOsqd/BJvuwih8n4TEHIHKSueuPTMg/0j2QLHzgmew8I+DtVxV01nElanR7FQww9n7D3/AEl+KKt6Rr5+TGGd0yZ4J4C/ELhrNQpGlQ7Z/jI7fD1nSwAAANgJZp6KtPSlNNapWgwqgbATJOtjhROjzPIz1mvqZWIiTKBERAEREAREQChG0874o8PafjNGLKlNi/dfGGHwM9HKEZmKlUtMlNOXtHCOLeEOK6Jz5Ceanb1mht0vEtOwa3QXBlPUDM+kXorcYKCR34ZpnGGqX6TUrhy/Bvx6jklHzcb7629qi5D3zUZWzVO4wTYNu6mfRL8C0TdaE+kxf7u6DOf6PX/2yv6FFn6nfyfP9WscALXXYf5a2krT6fiWqbl0+ivYserLgTvKcB0KdKE+Qkqnh2mq+5Wo+UzPBlFdeoWzm/g7wXatyavidYZlOVVhkA/vOo1ryqBKqiqMKMS6bkQoWkaWTJWR7YiIkysREQBERAEREAREQBERAEREAREQBERAEREAREQBERAEREA//9k=" + } + } + ] + }, + { + "role": "user", + "content": "What fruit is in this image? Answer with just the fruit name, nothing else." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-9dfb21780c2b104a","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-9dfb21780c2b104a","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"apple"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9dfb21780c2b104a","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_image_attachment__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment__conversation_stream.snap new file mode 100644 index 000000000..1c0a2594d --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment__conversation_stream.snap @@ -0,0 +1,237 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "What fruit is in this image? Answer with just the fruit name, nothing else." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "apple" + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_image_attachment__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment__raw_events.snap new file mode 100644 index 000000000..361c2b487 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_image_attachment__raw_events.snap @@ -0,0 +1,26 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Message( + "apple", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_model_details.yml b/crates/jp_llm/tests/fixtures/vllm/test_model_details.yml new file mode 100644 index 000000000..a604c3d4c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_model_details.yml @@ -0,0 +1,64 @@ +when: + path: /v1/models + method: GET +then: + status: 200 + header: + - name: content-type + value: application/json + json_body_str: >- + { + "object": "list", + "data": [ + { + "id": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "object": "model", + "created": 1789995687, + "owned_by": "flash-next-rtx", + "root": "/model", + "parent": null, + "max_model_len": 131072, + "permission": [ + { + "id": "modelperm-9d01a509d01965fe", + "object": "model_permission", + "created": 1789995687, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + }, + { + "id": "rtx6000", + "object": "model", + "created": 1789995687, + "owned_by": "flash-next-rtx", + "root": "/model", + "parent": null, + "max_model_len": 131072, + "permission": [ + { + "id": "modelperm-a8e7c2ee7f0f1f25", + "object": "model_permission", + "created": 1789995687, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + } + ] + } diff --git a/crates/jp_llm/tests/fixtures/vllm/test_model_details__model_details.snap b/crates/jp_llm/tests/fixtures/vllm/test_model_details__model_details.snap new file mode 100644 index 000000000..767bdb6e0 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_model_details__model_details.snap @@ -0,0 +1,25 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + ModelDetails { + id: ModelIdConfig { + provider: Vllm, + name: Name( + "Qwen/Qwen3.8-Flash-Next-NVFP4", + ), + }, + display_name: None, + context_window: Some( + 131072, + ), + max_output_tokens: None, + reasoning: None, + knowledge_cutoff: None, + deprecated: None, + structured_output: None, + prefill: None, + features: [], + }, +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_models.yml b/crates/jp_llm/tests/fixtures/vllm/test_models.yml new file mode 100644 index 000000000..961629c1b --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_models.yml @@ -0,0 +1,64 @@ +when: + path: /v1/models + method: GET +then: + status: 200 + header: + - name: content-type + value: application/json + json_body_str: >- + { + "object": "list", + "data": [ + { + "id": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "object": "model", + "created": 1789995687, + "owned_by": "flash-next-rtx", + "root": "/model", + "parent": null, + "max_model_len": 131072, + "permission": [ + { + "id": "modelperm-ad43be73d71012cf", + "object": "model_permission", + "created": 1789995687, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + }, + { + "id": "rtx6000", + "object": "model", + "created": 1789995687, + "owned_by": "flash-next-rtx", + "root": "/model", + "parent": null, + "max_model_len": 131072, + "permission": [ + { + "id": "modelperm-9e88e43fa449c04f", + "object": "model_permission", + "created": 1789995687, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + } + ] + } diff --git a/crates/jp_llm/tests/fixtures/vllm/test_models__models.snap b/crates/jp_llm/tests/fixtures/vllm/test_models__models.snap new file mode 100644 index 000000000..d248022e1 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_models__models.snap @@ -0,0 +1,44 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + ModelDetails { + id: ModelIdConfig { + provider: Vllm, + name: Name( + "Qwen/Qwen3.8-Flash-Next-NVFP4", + ), + }, + display_name: None, + context_window: Some( + 131072, + ), + max_output_tokens: None, + reasoning: None, + knowledge_cutoff: None, + deprecated: None, + structured_output: None, + prefill: None, + features: [], + }, + ModelDetails { + id: ModelIdConfig { + provider: Vllm, + name: Name( + "rtx6000", + ), + }, + display_name: None, + context_window: Some( + 131072, + ), + max_output_tokens: None, + reasoning: None, + knowledge_cutoff: None, + deprecated: None, + structured_output: None, + prefill: None, + features: [], + }, +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation.snap b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation.snap new file mode 100644 index 000000000..ee38f95b1 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation.snap @@ -0,0 +1,115 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "Test received! How can I help you today?", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Reasoning { + reasoning: "We need answer to user: \"Repeat my previous message\". Need maybe mention previous user message was \"Test message\". Need likely respond exactly? Could say Your previous message was: \"Test message\". Need ensure no private chain of thought? Final concise.\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "Your previous message was: “Test message”", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-b4ac3d9a0f62fbdb", + name: "run_me", + arguments: { + "foo": String("Test message"), + "bar": Array [ + String("foo"), + ], + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The secret code is: 42", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Reasoning { + reasoning: "We need answer to user: \"What was the result of the previous tool call?\" Need likely state result was The secret code is: 42. Need maybe concise. Also consider \"Previous\" tool call was run_me returned secret code. We can answer. But hidden? User asks result of previous tool call, okay tool output visible? In this interaction, tool output is visible? The instruction likely test. Need not mention internal. Need final.\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The previous tool call returned: `The secret code is: 42`", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation.yml b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation.yml new file mode 100644 index 000000000..ae02f672c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation.yml @@ -0,0 +1,463 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Test message" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-80bc1fed31ae854f","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-80bc1fed31ae854f","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"Test"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80bc1fed31ae854f","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" received"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80bc1fed31ae854f","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"! How"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80bc1fed31ae854f","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" can I help you"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80bc1fed31ae854f","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" today?"},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Test message" + }, + { + "role": "assistant", + "content": "Test received! How can I help you today?" + }, + { + "role": "user", + "content": "Repeat my previous message" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": true + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"We"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" need answer to"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" user: \"Repeat"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" my previous message"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\". Need maybe"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" mention previous user"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" message was \"Test"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" message\". Need likely"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" respond"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" exactly"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Could"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" say"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Your"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" previous message was:"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \"Test message\"."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Need ensure"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" no private"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" chain"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" of thought? Final"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" concise.\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n\nYour previous message"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" was: “"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-87d7a63e33df5b88","object":"chat.completion.chunk","created":1789995681,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"Test message”"},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Test message" + }, + { + "role": "assistant", + "content": "Test received! How can I help you today?" + }, + { + "role": "user", + "content": "Repeat my previous message" + }, + { + "role": "assistant", + "reasoning_content": "We need answer to user: \"Repeat my previous message\". Need maybe mention previous user message was \"Test message\". Need likely respond exactly? Could say Your previous message was: \"Test message\". Need ensure no private chain of thought? Final concise.\n", + "content": "Your previous message was: “Test message”" + }, + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "required" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-9876523863b00ff7","object":"chat.completion.chunk","created":1789995687,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-9876523863b00ff7","object":"chat.completion.chunk","created":1789995687,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-b4ac3d9a0f62fbdb","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9876523863b00ff7","object":"chat.completion.chunk","created":1789995687,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9876523863b00ff7","object":"chat.completion.chunk","created":1789995687,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Test message\", \"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9876523863b00ff7","object":"chat.completion.chunk","created":1789995687,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"foo\"]}"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9876523863b00ff7","object":"chat.completion.chunk","created":1789995687,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Test message" + }, + { + "role": "assistant", + "content": "Test received! How can I help you today?" + }, + { + "role": "user", + "content": "Repeat my previous message" + }, + { + "role": "assistant", + "reasoning_content": "We need answer to user: \"Repeat my previous message\". Need maybe mention previous user message was \"Test message\". Need likely respond exactly? Could say Your previous message was: \"Test message\". Need ensure no private chain of thought? Final concise.\n", + "content": "Your previous message was: “Test message”" + }, + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "chatcmpl-tool-b4ac3d9a0f62fbdb", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"Test message\",\"bar\":[\"foo\"]}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-b4ac3d9a0f62fbdb", + "content": "The secret code is: 42" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-8ba76cf0cdd4e48a","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-8ba76cf0cdd4e48a","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8ba76cf0cdd4e48a","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" secret code is:"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8ba76cf0cdd4e48a","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" 42"},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Test message" + }, + { + "role": "assistant", + "content": "Test received! How can I help you today?" + }, + { + "role": "user", + "content": "Repeat my previous message" + }, + { + "role": "assistant", + "reasoning_content": "We need answer to user: \"Repeat my previous message\". Need maybe mention previous user message was \"Test message\". Need likely respond exactly? Could say Your previous message was: \"Test message\". Need ensure no private chain of thought? Final concise.\n", + "content": "Your previous message was: “Test message”" + }, + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "tool_calls": [ + { + "id": "chatcmpl-tool-b4ac3d9a0f62fbdb", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"Test message\",\"bar\":[\"foo\"]}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-b4ac3d9a0f62fbdb", + "content": "The secret code is: 42" + }, + { + "role": "assistant", + "content": "The secret code is: 42" + }, + { + "role": "user", + "content": "What was the result of the previous tool call?" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": true + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"We"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" need answer to"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" user: \"What"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" was the result of"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" the previous tool call"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"?\" Need likely state"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" result was"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" secret code is:"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" 42."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Need maybe concise"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". Also"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" consider"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"Previous"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\" tool"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" call was"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" run_me returned"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" secret"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" code. We"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" can answer. But"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" hidden"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"? User"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" asks result of"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" previous tool call,"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" okay"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" tool"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" output"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" visible?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" In"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" this"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" interaction"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", tool output is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" visible"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"? The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" instruction"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" likely"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" test"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". Need not"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" mention"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" internal"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". Need"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" final.\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n\nThe previous"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool call returned:"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The secret code is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":": 42"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-90ad77f75e96598f","object":"chat.completion.chunk","created":1789995689,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"`"},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation__conversation_stream.snap new file mode 100644 index 000000000..848f48827 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation__conversation_stream.snap @@ -0,0 +1,357 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": { + "effort": "low", + "exclude": false + }, + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": "off" + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Test message" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "Test received! How can I help you today?" + }, + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": { + "effort": "low", + "exclude": false + } + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Repeat my previous message" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "reasoning": "We need answer to user: \"Repeat my previous message\". Need maybe mention previous user message was \"Test message\". Need likely respond exactly? Could say Your previous message was: \"Test message\". Need ensure no private chain of thought? Final concise.\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "Your previous message was: “Test message”" + }, + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": "off" + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-b4ac3d9a0f62fbdb", + "name": "run_me", + "arguments": { + "foo": "VGVzdCBtZXNzYWdl", + "bar": [ + "Zm9v" + ] + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-b4ac3d9a0f62fbdb", + "content": "VGhlIHNlY3JldCBjb2RlIGlzOiA0Mg==", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The secret code is: 42" + }, + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": { + "effort": "low", + "exclude": false + } + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "What was the result of the previous tool call?" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "reasoning": "We need answer to user: \"What was the result of the previous tool call?\" Need likely state result was The secret code is: 42. Need maybe concise. Also consider \"Previous\" tool call was run_me returned secret code. We can answer. But hidden? User asks result of previous tool call, okay tool output visible? In this interaction, tool output is visible? The instruction likely test. Need not mention internal. Need final.\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The previous tool call returned: `The secret code is: 42`" + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation__raw_events.snap new file mode 100644 index 000000000..686fdf0dc --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_multi_turn_conversation__raw_events.snap @@ -0,0 +1,693 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Message( + "Test r", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "eceiv", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ed! How can I h", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "elp you", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + " today?", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 0, + part: Reasoning( + "We", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " need answer to", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " user: \"Repeat", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " my previous message", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\". Need maybe", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " mention previous user", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " message was \"Test", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " message\". Need likely", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " respond", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " exactly", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Could", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " say", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Your", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " previous message was:", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"Test message\".", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Need ensure", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " no private", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " chain", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " of thought? Final", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " concise.\n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "Your previous ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "message w", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "as: “Test mes", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "sage”", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-b4ac3d9a0f62fbdb", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"Test message\", \"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "[\"foo\"]}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "The secret c", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ode", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + " is: 42", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 0, + part: Reasoning( + "We", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " need answer to", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " user: \"What", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " was the result of", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " the previous tool call", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "?\" Need likely state", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " result was", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " The", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " secret code is:", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " 42.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Need maybe concise", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". Also", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " consider", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "Previous", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\" tool", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " call was", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " run_me returned", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " secret", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " code. We", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " can answer. But", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " hidden", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "? User", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " asks result of", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " previous tool call,", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " okay", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " tool", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " output", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " visible?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " In", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " this", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " interaction", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", tool output is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " visible", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "? The", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " instruction", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " likely", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " test", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". Need not", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " mention", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " internal", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". Need", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " final.\n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "The p", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "revious tool call re", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "tu", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "rned: `The secret ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "code", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + " ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "is: 42`", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_structured_output.snap b/crates/jp_llm/tests/fixtures/vllm/test_structured_output.snap new file mode 100644 index 000000000..3d0acc45c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_structured_output.snap @@ -0,0 +1,26 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Structured { + data: Object { + "titles": Array [ + String("Conversation Title Generation"), + ], + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_structured_output.yml b/crates/jp_llm/tests/fixtures/vllm/test_structured_output.yml new file mode 100644 index 000000000..c8e374c4c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_structured_output.yml @@ -0,0 +1,73 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Generate a title for this conversation." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + }, + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "structured_output", + "schema": { + "type": "object", + "required": [ + "titles" + ], + "additionalProperties": false, + "properties": { + "titles": { + "type": "array", + "items": { + "type": "string", + "description": "A concise, descriptive title for the conversation" + }, + "minItems": 1, + "maxItems": 1 + } + } + }, + "strict": true + } + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"{"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" \"titles"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\": ["},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n \"Conversation"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" Title"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" Generation"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n ]\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-80e2cf96d611e363","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"}"},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_structured_output__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_structured_output__conversation_stream.snap new file mode 100644 index 000000000..9c8d14457 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_structured_output__conversation_stream.snap @@ -0,0 +1,259 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Generate a title for this conversation.", + "schema": { + "type": "object", + "required": [ + "titles" + ], + "additionalProperties": false, + "properties": { + "titles": { + "type": "array", + "items": { + "type": "string", + "description": "A concise, descriptive title for the conversation" + }, + "minItems": 1, + "maxItems": 1 + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "data": { + "titles": [ + "Conversation Title Generation" + ] + } + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_structured_output__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_structured_output__raw_events.snap new file mode 100644 index 000000000..2494af6ec --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_structured_output__raw_events.snap @@ -0,0 +1,82 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Structured( + "{\n ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "\"tit", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "les\": [\n \"Conve", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "rsatio", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "n Title Gen", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "e", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "ratio", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Structured( + "\"\n ]\n}", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto.snap new file mode 100644 index 000000000..7e9b54392 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto.snap @@ -0,0 +1,56 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "I'll run the tool with some arguments:\n\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-b5698c9449264def", + name: "run_me", + arguments: { + "foo": String("Hello, world!"), + "bar": Array [ + String("foo"), + ], + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The tool ran successfully and returned **\"working!\"**.", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto.yml b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto.yml new file mode 100644 index 000000000..8aa4358bd --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto.yml @@ -0,0 +1,148 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "auto" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"'ll run"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" the tool with some"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" arguments"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":":\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-b5698c9449264def","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Hello, world!\", \"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"foo\"]}"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-aa2c03c35c9017b6","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "content": "I'll run the tool with some arguments:\n\n", + "tool_calls": [ + { + "id": "chatcmpl-tool-b5698c9449264def", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"Hello, world!\",\"bar\":[\"foo\"]}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-b5698c9449264def", + "content": "working!" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-b207a04446a93105","object":"chat.completion.chunk","created":1789995680,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-b207a04446a93105","object":"chat.completion.chunk","created":1789995680,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b207a04446a93105","object":"chat.completion.chunk","created":1789995680,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool ran successfully and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b207a04446a93105","object":"chat.completion.chunk","created":1789995680,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" returned **\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b207a04446a93105","object":"chat.completion.chunk","created":1789995680,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"working!\"**."},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto__conversation_stream.snap new file mode 100644 index 000000000..bec9a5e0d --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto__conversation_stream.snap @@ -0,0 +1,261 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "I'll run the tool with some arguments:\n\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-b5698c9449264def", + "name": "run_me", + "arguments": { + "foo": "SGVsbG8sIHdvcmxkIQ==", + "bar": [ + "Zm9v" + ] + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-b5698c9449264def", + "content": "d29ya2luZyE=", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The tool ran successfully and returned **\"working!\"**." + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto__raw_events.snap new file mode 100644 index 000000000..359600205 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_auto__raw_events.snap @@ -0,0 +1,136 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Message( + "I", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "'ll run the tool wi", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "th some ar", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "gum", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ents:\n\n", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-b5698c9449264def", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"Hello, world!\", \"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "[\"foo\"]}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "The tool ran successfu", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "lly and retur", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ned **\"worki", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ng!\"**.", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function.snap new file mode 100644 index 000000000..54bfea2d4 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function.snap @@ -0,0 +1,56 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "I'll run the tool with some arguments:\n\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-9471b01d74e3a119", + name: "run_me", + arguments: { + "foo": String("Hello, world!"), + "bar": Array [ + String("foo"), + ], + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The tool executed successfully and returned \"working!\"", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function.yml b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function.yml new file mode 100644 index 000000000..be507d306 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function.yml @@ -0,0 +1,150 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "required" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"'ll run the tool"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" with some"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" arguments"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":":\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-9471b01d74e3a119","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Hello, world!\", \"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"foo\"]}"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-b3ae47acfa3680c0","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "content": "I'll run the tool with some arguments:\n\n", + "tool_calls": [ + { + "id": "chatcmpl-tool-9471b01d74e3a119", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"Hello, world!\",\"bar\":[\"foo\"]}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-9471b01d74e3a119", + "content": "working!" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-8905494e87d6b92b","object":"chat.completion.chunk","created":1789995683,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-8905494e87d6b92b","object":"chat.completion.chunk","created":1789995683,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8905494e87d6b92b","object":"chat.completion.chunk","created":1789995683,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool executed"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8905494e87d6b92b","object":"chat.completion.chunk","created":1789995683,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" successfully and returned \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8905494e87d6b92b","object":"chat.completion.chunk","created":1789995683,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"working!\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8905494e87d6b92b","object":"chat.completion.chunk","created":1789995683,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function__conversation_stream.snap new file mode 100644 index 000000000..35ee6d0e7 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function__conversation_stream.snap @@ -0,0 +1,261 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "I'll run the tool with some arguments:\n\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-9471b01d74e3a119", + "name": "run_me", + "arguments": { + "foo": "SGVsbG8sIHdvcmxkIQ==", + "bar": [ + "Zm9v" + ] + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-9471b01d74e3a119", + "content": "d29ya2luZyE=", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The tool executed successfully and returned \"working!\"" + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function__raw_events.snap new file mode 100644 index 000000000..37c7beaee --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_function__raw_events.snap @@ -0,0 +1,136 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Message( + "I'll run t", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "he tool wi", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "th some ar", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "gum", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ents:\n\n", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-9471b01d74e3a119", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"Hello, world!\", \"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "[\"foo\"]}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "The tool e", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "xecuted successfully and ret", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "urned \"wo", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "rking!\"", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning.snap new file mode 100644 index 000000000..bf409fac8 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning.snap @@ -0,0 +1,54 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Reasoning { + reasoning: "The user wants me to run the tool `run_me` with whatever arguments I want.\n\nRequired parameters: `foo` (string or null) and `bar` (array of strings, enum [\"foo\", \"bar\"]). Also `foo` is required, so it can't be null probably? Type allows null but it's required. I'll pass a string.\n\n`bar` must be an array with items from enum [\"foo\", \"bar\"]. I'll pass [\"foo\", \"bar\"].\n\nWait, `bar` has enum [\"foo\"] at the array level? The schema says `\"bar\": {\"type\": [\"string\", \"array\"], \"enum\": [\"foo\"], \"items\": {\"type\": \"string\", \"enum\": [\"foo\", \"bar\"]}}`. Hmm, if enum is [\"foo\"] then if it's a string it must be \"foo\". If it's an array... enums apply to exact values, so an array wouldn't match enum [\"foo\"]? But strict mode validation might only apply items. This is tricky. Safest choice: pass `bar` as the string \"foo\"? But enum allows \"foo\" string which matches enum. However items would not apply if it's a string. Actually if type is [\"string\", \"array\"] and enum is [\"foo\"], the string \"foo\" satisfies both. If I pass an array, it might fail enum validation.\n\nAlternatively, pass an array [\"foo\"] — risk of enum mismatch at array level. Hmm.\n\nGiven strictness, string \"foo\" seems safest for matching enum, but then items constraint... a string type wouldn't violate items since items only applies to arrays. But if validator checks array enum, [\"foo\"] fails. So safest: \"foo\" as string? But wait, if validator is strict about required types... Let's just do both? I can only call once per chosen arg set, but I could make two calls to test both. The user said run the tool with whatever args. I could call it twice in parallel with different formats. But that might be excessive.\n\nI'll just call once with bar as [\"foo\"]? Hmm.\n\nActually let's reason: In many of these test setups, the schema intentionally has both string/array types with enum. The enum [\"foo\"] likely applies only when string is chosen; but JSON Schema enums always apply regardless of type. So an array value would fail. However some validators (like Pydantic strict) would validate enum only for string branch... In JSON Schema, `enum` is evaluated on the instance value directly, so `[\"foo\"] != \"foo\"` → fail.\n\nBut the schema also gives `items` with enum, which suggests array usage is intended. Ambiguous.\n\nTo be safe and satisfy both interpretations, could I pass \"foo\" as a string? That fails items? No, items doesn't apply to strings. So \"foo\" string passes: type string allowed, enum matched, items ignored. That's the safest.\n\nHmm but if the validator requires array type strictly... type is [\"string\",\"array\"], so string is allowed.\n\nI'll go with foo: \"hello\" (or null? required so use string), bar: \"foo\".\n\nActually could I pass foo as null? It's required; passing null might be allowed since type allows null. But let's just use a string.\n\nMake one call. Then report result.\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-ac050526c1a6a621", + name: "run_me", + arguments: { + "foo": String("hello world"), + "bar": String("foo"), + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The tool ran successfully! I called `run_me` with:\n\n- **foo**: `\"hello world\"`\n- **bar**: `\"foo\"`\n\nAnd the tool responded with **\"working!\"** — so it looks like everything executed as expected.", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning.yml b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning.yml new file mode 100644 index 000000000..524be35c2 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning.yml @@ -0,0 +1,800 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "auto" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" user wants me to"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" run the tool `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"run_me` with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" whatever"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" arguments I want."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\n\nRequired"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" parameters: `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo` (string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" or null) and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" `bar` ("},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" of strings, enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\", \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar\"])."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Also `foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"` is required,"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" so"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" can't be null"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" probably"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" allows"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" null but"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s required. I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'ll pass"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" a string.\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"`bar` must"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" be an array with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items from enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\", \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar\"]. I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'ll pass [\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\", \"bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"]."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\n\nWait"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar` has"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\"]"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" at the array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" level? The schema"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" says `\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar\": {\"type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\": [\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"string\", \"array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"], \"enum\":"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\"], \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"items\": {\"type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\": \"string\","},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \"enum\": [\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\", \"bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"]}}`."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Hmm, if"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is [\"foo\"]"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" then"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" if"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s a string it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" must be \"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\". If it's"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" an array..."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enums"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" apply"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" to exact"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" values,"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" so an"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array wouldn"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'t match enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\"]?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" But"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" strict"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" mode validation"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" might only"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" apply"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". This"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is tricky"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". Saf"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"est choice"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":": pass `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar` as the"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string \"foo\"?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" But enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" allows"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\" string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" which"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" matches enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" However"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" would"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" not"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" apply if"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" it's a string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". Actually"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" if"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is [\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"string\", \"array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"] and enum is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\"], the"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \"foo\" satisfies"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" both. If"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" I pass an"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array, it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" might fail enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" validation.\n\nAlternatively"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":","},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" pass"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" an array [\"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"] — risk"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" of enum mismatch at"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" level. Hmm."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\n\nGiven"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" strict"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"ness, string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \"foo\" seems"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" safest for matching enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":","},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" but then"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" constraint"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"... a"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" wouldn"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'t violate"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items since"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items only applies to"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" arrays. But if"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" validator"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" checks"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":","},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\"] fails"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". So"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" safest"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":": \"foo\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" as string?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" But wait"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", if validator"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" strict about required"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" types"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"..."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Let"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" just do"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" both"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"? I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" can only call once"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" per"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" chosen"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" arg"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" set"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":","},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" but I could"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" make"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" two calls to"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" test both"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" user"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" said"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" run the tool with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" whatever args"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". I could"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" call"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" twice in parallel with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" different formats"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" But that"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" might be excessive"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":".\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'ll just call once"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" with bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" as"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\"]? Hmm"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":".\n\nActually"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" let"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" reason"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":": In"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" many of"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" these test"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" setups, the schema"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" intentionally"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" has both"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"/array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" types"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" The enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\"] likely"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" applies"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" only"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" when"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" chosen"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":";"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" but JSON"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Schema enums"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" always"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" apply regardless"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" of type. So"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" an"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array value"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" would fail. However"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" some"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" validators"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" (like Py"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"dantic strict"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":") would"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" validate"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" only"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" for"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" branch"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"... In"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" JSON"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Schema, `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"` is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" evaluated"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" on"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" the instance"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" value directly"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":","},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" so `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"[\"foo\"] !="},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\"` → fail"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":".\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"But the schema"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" also gives"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" `items` with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum, which suggests"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array usage"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is intended. Amb"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"iguous.\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"To be safe and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" satisfy"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" both"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" interpretations"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", could"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" I pass \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" as a"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string? That fails"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"?"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" No, items doesn"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'t apply to strings"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". So"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\" string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" passes"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":":"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" type string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" allowed"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", enum matched"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", items ignored"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":". That's the"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" safest."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\n\nHmm"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" but if the validator"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" requires"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" strictly"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"... type"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"string\",\"array"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"],"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" so string is"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" allowed.\n\nI"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'ll go"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" with foo: \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"hello\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" ("},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"or null? required"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" so"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" use"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string), bar:"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \"foo\".\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"Actually could"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" pass foo as null"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"? It"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s required;"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" passing"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" null might"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" be allowed since"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" type allows"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" null. But let"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s just use a"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" string.\n\nMake"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" one call."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Then"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" report"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" result.\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-ac050526c1a6a621","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"hello world\", \"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"foo\"}"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-95699bedb8fdae74","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "reasoning_content": "The user wants me to run the tool `run_me` with whatever arguments I want.\n\nRequired parameters: `foo` (string or null) and `bar` (array of strings, enum [\"foo\", \"bar\"]). Also `foo` is required, so it can't be null probably? Type allows null but it's required. I'll pass a string.\n\n`bar` must be an array with items from enum [\"foo\", \"bar\"]. I'll pass [\"foo\", \"bar\"].\n\nWait, `bar` has enum [\"foo\"] at the array level? The schema says `\"bar\": {\"type\": [\"string\", \"array\"], \"enum\": [\"foo\"], \"items\": {\"type\": \"string\", \"enum\": [\"foo\", \"bar\"]}}`. Hmm, if enum is [\"foo\"] then if it's a string it must be \"foo\". If it's an array... enums apply to exact values, so an array wouldn't match enum [\"foo\"]? But strict mode validation might only apply items. This is tricky. Safest choice: pass `bar` as the string \"foo\"? But enum allows \"foo\" string which matches enum. However items would not apply if it's a string. Actually if type is [\"string\", \"array\"] and enum is [\"foo\"], the string \"foo\" satisfies both. If I pass an array, it might fail enum validation.\n\nAlternatively, pass an array [\"foo\"] — risk of enum mismatch at array level. Hmm.\n\nGiven strictness, string \"foo\" seems safest for matching enum, but then items constraint... a string type wouldn't violate items since items only applies to arrays. But if validator checks array enum, [\"foo\"] fails. So safest: \"foo\" as string? But wait, if validator is strict about required types... Let's just do both? I can only call once per chosen arg set, but I could make two calls to test both. The user said run the tool with whatever args. I could call it twice in parallel with different formats. But that might be excessive.\n\nI'll just call once with bar as [\"foo\"]? Hmm.\n\nActually let's reason: In many of these test setups, the schema intentionally has both string/array types with enum. The enum [\"foo\"] likely applies only when string is chosen; but JSON Schema enums always apply regardless of type. So an array value would fail. However some validators (like Pydantic strict) would validate enum only for string branch... In JSON Schema, `enum` is evaluated on the instance value directly, so `[\"foo\"] != \"foo\"` → fail.\n\nBut the schema also gives `items` with enum, which suggests array usage is intended. Ambiguous.\n\nTo be safe and satisfy both interpretations, could I pass \"foo\" as a string? That fails items? No, items doesn't apply to strings. So \"foo\" string passes: type string allowed, enum matched, items ignored. That's the safest.\n\nHmm but if the validator requires array type strictly... type is [\"string\",\"array\"], so string is allowed.\n\nI'll go with foo: \"hello\" (or null? required so use string), bar: \"foo\".\n\nActually could I pass foo as null? It's required; passing null might be allowed since type allows null. But let's just use a string.\n\nMake one call. Then report result.\n", + "tool_calls": [ + { + "id": "chatcmpl-tool-ac050526c1a6a621", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"hello world\",\"bar\":\"foo\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-ac050526c1a6a621", + "content": "working!" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool ran successfully!"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" called"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"run_me` with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":":"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n\n- **foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"**: `\"hello world"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\"`\n- **"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"bar**: `\"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\"`\n\nAnd"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" the"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool responded"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" with **\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"working!\"**"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" — so it looks"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" like everything"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" executed"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" as"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-84a0865ff7500de8","object":"chat.completion.chunk","created":1789995722,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" expected."},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning__conversation_stream.snap new file mode 100644 index 000000000..8760fdacd --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning__conversation_stream.snap @@ -0,0 +1,288 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": { + "effort": "low", + "exclude": false + } + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "reasoning": "The user wants me to run the tool `run_me` with whatever arguments I want.\n\nRequired parameters: `foo` (string or null) and `bar` (array of strings, enum [\"foo\", \"bar\"]). Also `foo` is required, so it can't be null probably? Type allows null but it's required. I'll pass a string.\n\n`bar` must be an array with items from enum [\"foo\", \"bar\"]. I'll pass [\"foo\", \"bar\"].\n\nWait, `bar` has enum [\"foo\"] at the array level? The schema says `\"bar\": {\"type\": [\"string\", \"array\"], \"enum\": [\"foo\"], \"items\": {\"type\": \"string\", \"enum\": [\"foo\", \"bar\"]}}`. Hmm, if enum is [\"foo\"] then if it's a string it must be \"foo\". If it's an array... enums apply to exact values, so an array wouldn't match enum [\"foo\"]? But strict mode validation might only apply items. This is tricky. Safest choice: pass `bar` as the string \"foo\"? But enum allows \"foo\" string which matches enum. However items would not apply if it's a string. Actually if type is [\"string\", \"array\"] and enum is [\"foo\"], the string \"foo\" satisfies both. If I pass an array, it might fail enum validation.\n\nAlternatively, pass an array [\"foo\"] — risk of enum mismatch at array level. Hmm.\n\nGiven strictness, string \"foo\" seems safest for matching enum, but then items constraint... a string type wouldn't violate items since items only applies to arrays. But if validator checks array enum, [\"foo\"] fails. So safest: \"foo\" as string? But wait, if validator is strict about required types... Let's just do both? I can only call once per chosen arg set, but I could make two calls to test both. The user said run the tool with whatever args. I could call it twice in parallel with different formats. But that might be excessive.\n\nI'll just call once with bar as [\"foo\"]? Hmm.\n\nActually let's reason: In many of these test setups, the schema intentionally has both string/array types with enum. The enum [\"foo\"] likely applies only when string is chosen; but JSON Schema enums always apply regardless of type. So an array value would fail. However some validators (like Pydantic strict) would validate enum only for string branch... In JSON Schema, `enum` is evaluated on the instance value directly, so `[\"foo\"] != \"foo\"` → fail.\n\nBut the schema also gives `items` with enum, which suggests array usage is intended. Ambiguous.\n\nTo be safe and satisfy both interpretations, could I pass \"foo\" as a string? That fails items? No, items doesn't apply to strings. So \"foo\" string passes: type string allowed, enum matched, items ignored. That's the safest.\n\nHmm but if the validator requires array type strictly... type is [\"string\",\"array\"], so string is allowed.\n\nI'll go with foo: \"hello\" (or null? required so use string), bar: \"foo\".\n\nActually could I pass foo as null? It's required; passing null might be allowed since type allows null. But let's just use a string.\n\nMake one call. Then report result.\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-ac050526c1a6a621", + "name": "run_me", + "arguments": { + "foo": "aGVsbG8gd29ybGQ=", + "bar": "Zm9v" + } + }, + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": "off" + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-ac050526c1a6a621", + "content": "d29ya2luZyE=", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The tool ran successfully! I called `run_me` with:\n\n- **foo**: `\"hello world\"`\n- **bar**: `\"foo\"`\n\nAnd the tool responded with **\"working!\"** — so it looks like everything executed as expected." + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning__raw_events.snap new file mode 100644 index 000000000..86ac8583c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_reasoning__raw_events.snap @@ -0,0 +1,2418 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 0, + part: Reasoning( + "The", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " user wants me to", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " run the tool `", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "run_me` with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " whatever", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " arguments I want.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\n\nRequired", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " parameters: `", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo` (string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " or null) and", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " `bar` (", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " of strings, enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\", \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar\"]).", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Also `foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "` is required,", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " so", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " can't be null", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " probably", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " allows", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " null but", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s required. I", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'ll pass", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " a string.\n\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "`bar` must", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " be an array with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items from enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\", \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar\"]. I", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'ll pass [\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\", \"bar", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"].", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\n\nWait", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", `", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar` has", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\"]", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " at the array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " level? The schema", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " says `\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar\": {\"type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\": [\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "string\", \"array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"], \"enum\":", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\"], \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "items\": {\"type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\": \"string\",", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"enum\": [\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\", \"bar", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"]}}`.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Hmm, if", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is [\"foo\"]", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " then", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " if", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s a string it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " must be \"foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\". If it's", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " an array...", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enums", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " apply", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " to exact", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " values,", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " so an", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array wouldn", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'t match enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\"]?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " But", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " strict", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " mode validation", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " might only", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " apply", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". This", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is tricky", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". Saf", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "est choice", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ": pass `", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar` as the", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string \"foo\"?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " But enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " allows", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\" string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " which", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " matches enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ".", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " However", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " would", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " not", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " apply if", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " it's a string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". Actually", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " if", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is [\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "string\", \"array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"] and enum is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\"], the", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"foo\" satisfies", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " both. If", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " I pass an", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array, it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " might fail enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " validation.\n\nAlternatively", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ",", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " pass", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " an array [\"foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"] — risk", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " of enum mismatch at", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " level. Hmm.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\n\nGiven", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " strict", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "ness, string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"foo\" seems", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " safest for matching enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ",", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " but then", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " constraint", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "... a", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " wouldn", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'t violate", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items since", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items only applies to", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " arrays. But if", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " validator", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " checks", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ",", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\"] fails", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". So", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " safest", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ": \"foo\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " as string?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " But wait", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", if validator", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " strict about required", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " types", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "...", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Let", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " just do", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " both", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "? I", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " can only call once", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " per", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " chosen", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " arg", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " set", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ",", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " but I could", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " make", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " two calls to", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " test both", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". The", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " user", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " said", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " run the tool with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " whatever args", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". I could", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " call", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " twice in parallel with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " different formats", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ".", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " But that", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " might be excessive", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ".\n\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "I", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'ll just call once", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " with bar", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " as", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\"]? Hmm", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ".\n\nActually", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " let", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " reason", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ": In", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " many of", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " these test", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " setups, the schema", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " intentionally", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " has both", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "/array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " types", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " The enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\"] likely", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " applies", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " only", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " when", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " chosen", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ";", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " but JSON", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Schema enums", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " always", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " apply regardless", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " of type. So", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " an", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array value", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " would fail. However", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " some", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " validators", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " (like Py", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "dantic strict", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ") would", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " validate", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " only", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " for", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " branch", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "... In", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " JSON", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Schema, `", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "` is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " evaluated", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " on", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " the instance", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " value directly", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ",", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " so `", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "[\"foo\"] !=", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\"` → fail", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ".\n\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "But the schema", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " also gives", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " `items` with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum, which suggests", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array usage", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is intended. Amb", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "iguous.\n\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "To be safe and", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " satisfy", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " both", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " interpretations", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", could", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " I pass \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " as a", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string? That fails", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "?", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " No, items doesn", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'t apply to strings", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". So", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\" string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " passes", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ":", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " type string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " allowed", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", enum matched", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", items ignored", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ". That's the", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " safest.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\n\nHmm", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " but if the validator", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " requires", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " strictly", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "... type", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"string\",\"array", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"],", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " so string is", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " allowed.\n\nI", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'ll go", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " with foo: \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "hello\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " (", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "or null? required", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " so", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " use", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string), bar:", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"foo\".\n\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "Actually could", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " I", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " pass foo as null", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "? It", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s required;", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " passing", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " null might", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " be allowed since", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " type allows", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " null. But let", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s just use a", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " string.\n\nMake", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " one call.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Then", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " report", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " result.\n", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-ac050526c1a6a621", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"hello world\", \"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"foo\"}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "The tool ran succes", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "sf", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ully! I", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + " c", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "alled `run_m", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "e", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "` with:\n\n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "- **foo**: `\"hell", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "o world", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\"`\n- **bar**", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + ": `\"foo", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\"`\n\n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "And the tool re", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "sponded w", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ith **\"work", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ing!\"** — so i", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "t looks like eve", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "rything e", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "xec", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "uted as ex", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "pected.", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning.snap new file mode 100644 index 000000000..419535000 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning.snap @@ -0,0 +1,56 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "I'll run the tool with some arguments:\n\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-95b70da4d5404eab", + name: "run_me", + arguments: { + "foo": String("Hello, world!"), + "bar": Array [ + String("foo"), + ], + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The tool ran successfully and returned **\"working!\"** — it seems to have executed with the arguments I provided:\n\n- **foo**: `\"Hello, world!\"`\n- **bar**: `[\"foo\"]`", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning.yml b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning.yml new file mode 100644 index 000000000..6f0ec5223 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning.yml @@ -0,0 +1,174 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "required" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"'ll run the tool"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" with some"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" arguments"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":":\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-95b70da4d5404eab","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Hello, world!\", \"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"foo\"]}"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-ac6cf978f4510c10","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "content": "I'll run the tool with some arguments:\n\n", + "tool_calls": [ + { + "id": "chatcmpl-tool-95b70da4d5404eab", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"Hello, world!\",\"bar\":[\"foo\"]}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-95b70da4d5404eab", + "content": "working!" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool ran successfully and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" returned **\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"working!\"**"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" —"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" it seems to have"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" executed"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" the arguments"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" I provided:\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"- **"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"**: `\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"Hello, world!\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"`\n- **"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"bar**: `[\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-8d78d36f65b6e95c","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"foo\"]`"},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning__conversation_stream.snap new file mode 100644 index 000000000..3ac05acb8 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning__conversation_stream.snap @@ -0,0 +1,261 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "I'll run the tool with some arguments:\n\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-95b70da4d5404eab", + "name": "run_me", + "arguments": { + "foo": "SGVsbG8sIHdvcmxkIQ==", + "bar": [ + "Zm9v" + ] + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-95b70da4d5404eab", + "content": "d29ya2luZyE=", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The tool ran successfully and returned **\"working!\"** — it seems to have executed with the arguments I provided:\n\n- **foo**: `\"Hello, world!\"`\n- **bar**: `[\"foo\"]`" + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning__raw_events.snap new file mode 100644 index 000000000..c6ff8be69 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_no_reasoning__raw_events.snap @@ -0,0 +1,227 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Message( + "I'll run t", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "he tool wi", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "th some ar", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "gum", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ents:\n\n", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-95b70da4d5404eab", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"Hello, world!\", \"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "[\"foo\"]}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "The tool ran successfu", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "lly and retur", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ned **\"work", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ing!", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\"** — it seems ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "to have e", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "xecut", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ed with the ar", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "guments I prov", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ided", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + ":\n\n", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "- **fo", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "o**: `\"Hello, ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "world!", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\"`\n- **bar", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "**: `[", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\"foo\"]`", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning.snap new file mode 100644 index 000000000..4a409392c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning.snap @@ -0,0 +1,57 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Reasoning { + reasoning: "The user wants me to run the tool with whatever arguments I want. Let me look at the schema:\n- foo: string or null, required\n- bar: string or array, required, enum [\"foo\"], items enum [\"foo\", \"bar\"]\n\nWait, bar has enum [\"foo\"] at the top level and items enum [\"foo\",\"bar\"]. That's a bit odd. If bar is a string, it must be \"foo\". If it's an array, items must be \"foo\" or \"bar\".\n\nLet me just call it with foo = \"hello\" and bar = [\"foo\", \"bar\"].\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-a561ff74956ad924", + name: "run_me", + arguments: { + "foo": String("hello world"), + "bar": Array [ + String("foo"), + String("bar"), + ], + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "I ran the `run_me` tool with:\n\n- **`foo`**: `\"hello world\"` (a string)\n- **`bar`**: `[\"foo\", \"bar\"]` (an array of valid enum items)\n\nAnd it returned: **working!** 🎉", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning.yml b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning.yml new file mode 100644 index 000000000..d362562bc --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning.yml @@ -0,0 +1,254 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": true + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "required" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" user wants me to"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" run the tool with"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" whatever"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" arguments I want."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" Let"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" me look"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" at the schema"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":":\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"- foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":": string or"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" null, required\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"- bar: string"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" or array, required"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", enum [\"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"],"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" [\"foo\", \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar\"]\n\nWait"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":", bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" has enum [\"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\"] at the top"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" level and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" enum [\"foo\",\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar\"]."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" That's a bit"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" odd. If bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" is a string,"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" it must be \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\". If it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"'s an array,"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" items must be \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"foo\" or \""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"bar\".\n\nLet"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" me just"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" call it"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" with foo ="},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" \"hello\" and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":" bar = [\"foo"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\", \"bar\"]."},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"reasoning":"\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-a561ff74956ad924","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"hello world\", \"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a1442d929e282f1c","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"foo\", \"bar\"]}"}}]},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "reasoning_content": "The user wants me to run the tool with whatever arguments I want. Let me look at the schema:\n- foo: string or null, required\n- bar: string or array, required, enum [\"foo\"], items enum [\"foo\", \"bar\"]\n\nWait, bar has enum [\"foo\"] at the top level and items enum [\"foo\",\"bar\"]. That's a bit odd. If bar is a string, it must be \"foo\". If it's an array, items must be \"foo\" or \"bar\".\n\nLet me just call it with foo = \"hello\" and bar = [\"foo\", \"bar\"].\n", + "tool_calls": [ + { + "id": "chatcmpl-tool-a561ff74956ad924", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"foo\":\"hello world\",\"bar\":[\"foo\",\"bar\"]}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-a561ff74956ad924", + "content": "working!" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" ran"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" the `"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"run_me` tool"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" with:"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\n\n- **`"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"foo`**: `\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"hello world"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\"` ("},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"a string)\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"- **`bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"`**: `[\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"foo\", \"bar"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"\"]` (an"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" array of valid enum"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" items"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":")\n\nAnd"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" it returned: **"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"working"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"!** 🎉"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-9ae2aad2c8bd589b","object":"chat.completion.chunk","created":1789995686,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning__conversation_stream.snap new file mode 100644 index 000000000..95f9c172f --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning__conversation_stream.snap @@ -0,0 +1,291 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": { + "effort": "low", + "exclude": false + } + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "reasoning": "The user wants me to run the tool with whatever arguments I want. Let me look at the schema:\n- foo: string or null, required\n- bar: string or array, required, enum [\"foo\"], items enum [\"foo\", \"bar\"]\n\nWait, bar has enum [\"foo\"] at the top level and items enum [\"foo\",\"bar\"]. That's a bit odd. If bar is a string, it must be \"foo\". If it's an array, items must be \"foo\" or \"bar\".\n\nLet me just call it with foo = \"hello\" and bar = [\"foo\", \"bar\"].\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-a561ff74956ad924", + "name": "run_me", + "arguments": { + "foo": "aGVsbG8gd29ybGQ=", + "bar": [ + "Zm9v", + "YmFy" + ] + } + }, + { + "type": "config_delta", + "timestamp": "2020-01-01 00:00:00.0", + "delta": { + "assistant": { + "model": { + "parameters": { + "reasoning": "off" + } + } + } + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-a561ff74956ad924", + "content": "d29ya2luZyE=", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "I ran the `run_me` tool with:\n\n- **`foo`**: `\"hello world\"` (a string)\n- **`bar`**: `[\"foo\", \"bar\"]` (an array of valid enum items)\n\nAnd it returned: **working!** 🎉" + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning__raw_events.snap new file mode 100644 index 000000000..0ea139a24 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_required_reasoning__raw_events.snap @@ -0,0 +1,500 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 0, + part: Reasoning( + "The", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " user wants me to", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " run the tool with", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " whatever", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " arguments I want.", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " Let", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " me look", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " at the schema", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ":\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "- foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ": string or", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " null, required\n", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "- bar: string", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " or array, required", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", enum [\"foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"],", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items enum", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " [\"foo\", \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar\"]\n\nWait", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + ", bar", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " has enum [\"foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\"] at the top", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " level and", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " enum [\"foo\",\"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar\"].", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " That's a bit", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " odd. If bar", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " is a string,", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " it must be \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\". If it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "'s an array,", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " items must be \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "foo\" or \"", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "bar\".\n\nLet", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " me just", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " call it", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " with foo =", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " \"hello\" and", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + " bar = [\"foo", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\", \"bar\"].", + ), + metadata: {}, + }, + Part { + index: 0, + part: Reasoning( + "\n", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-a561ff74956ad924", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"hello world\", \"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "[\"foo\", \"bar\"]}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "I ra", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "n the `run_m", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "e` too", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "l with:", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\n\n- **`foo", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "`**: `\"hell", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "o wo", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "rld\"` (a s", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "tring)\n-", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + " **`bar`", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "**: `[\"foo", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\", \"bar", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "\"]` (an array of val", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "id enu", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "m item", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "s)\n\nAnd it retur", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ned: **", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "working!", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "** 🎉", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream.snap new file mode 100644 index 000000000..5cafa765c --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream.snap @@ -0,0 +1,57 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "I'll call the tool with some values that fit its schema:\n\n", + }, + ), + metadata: {}, + }, + ), + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ToolCallRequest( + ToolCallRequest { + id: "chatcmpl-tool-8bef0cd026442499", + name: "run_me", + arguments: { + "bar": Array [ + String("foo"), + String("bar"), + ], + "foo": String("Hello, tool!"), + }, + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], + [ + Flushed( + ConversationEvent { + timestamp: 2020-01-01 0:00:00.0 +00, + kind: ChatResponse( + Message { + message: "The tool ran successfully and returned **\"working!\"**", + }, + ), + metadata: {}, + }, + ), + Finished( + Completed, + ), + ], +] diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream.yml b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream.yml new file mode 100644 index 000000000..895cf274b --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream.yml @@ -0,0 +1,152 @@ +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + }, + "tools": [ + { + "type": "function", + "function": { + "name": "run_me", + "description": "", + "parameters": { + "type": "object", + "properties": { + "foo": { + "type": [ + "string", + "null" + ], + "default": "foo" + }, + "bar": { + "type": [ + "string", + "array" + ], + "enum": [ + "foo" + ], + "items": { + "type": "string", + "enum": [ + "foo", + "bar" + ] + } + } + }, + "required": [ + "foo", + "bar" + ], + "additionalProperties": false + }, + "strict": true + } + } + ], + "tool_choice": "auto" + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"I"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"'ll call"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" the tool with some"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" values"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" that"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" fit its"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" schema:\n\n"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"id":"chatcmpl-tool-8bef0cd026442499","type":"function","index":0,"function":{"name":"run_me"}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\"bar\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"[\"foo\", \"bar\"], \"foo\": "}}]},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-92e1cd92185510f2","object":"chat.completion.chunk","created":1789995677,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"Hello, tool!\"}"}}]},"logprobs":null,"finish_reason":"tool_calls","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + +--- +when: + path: /v1/chat/completions + method: POST + json_body_str: >- + { + "model": "Qwen/Qwen3.8-Flash-Next-NVFP4", + "messages": [ + { + "role": "user", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "role": "assistant", + "content": "I'll call the tool with some values that fit its schema:\n\n", + "tool_calls": [ + { + "id": "chatcmpl-tool-8bef0cd026442499", + "type": "function", + "function": { + "name": "run_me", + "arguments": "{\"bar\":[\"foo\",\"bar\"],\"foo\":\"Hello, tool!\"}" + } + } + ] + }, + { + "role": "tool", + "tool_call_id": "chatcmpl-tool-8bef0cd026442499", + "content": "working!" + } + ], + "stream": true, + "chat_template_kwargs": { + "enable_thinking": false + } + } +then: + status: 200 + header: + - name: content-type + value: text/event-stream; charset=utf-8 + body: |+ + data: {"id":"chatcmpl-a666d064cf5965d2","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"role":"assistant","content":""},"logprobs":null,"finish_reason":null}],"prompt_token_ids":null,"prompt_text":null} + + data: {"id":"chatcmpl-a666d064cf5965d2","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"The"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a666d064cf5965d2","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" tool ran successfully and"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a666d064cf5965d2","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":" returned **\""},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a666d064cf5965d2","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{"content":"working!\"**"},"logprobs":null,"finish_reason":null,"token_ids":null}]} + + data: {"id":"chatcmpl-a666d064cf5965d2","object":"chat.completion.chunk","created":1789995682,"model":"Qwen/Qwen3.8-Flash-Next-NVFP4","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop","stop_reason":null,"token_ids":null}]} + + + data: [DONE] + diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream__conversation_stream.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream__conversation_stream.snap new file mode 100644 index 000000000..d3a047020 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream__conversation_stream.snap @@ -0,0 +1,262 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +{ + "base_config": { + "inherit": false, + "config_load_paths": [], + "extends": [ + "config.d/**/*" + ], + "assistant": { + "system_prompt": "You are a helpful assistant.", + "system_prompt_sections": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "instructions": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "tool_choice": "auto", + "model": { + "id": { + "provider": "vllm", + "name": "test" + }, + "parameters": { + "reasoning": "off", + "stop_words": [], + "other": {} + } + }, + "request": { + "max_retries": 5, + "base_backoff_ms": 1000, + "max_backoff_secs": 60, + "stream_idle_timeout_secs": 60, + "max_response_bytes": 1048576, + "cache": true + } + }, + "conversation": { + "title": { + "generate": { + "auto": false + }, + "from_heading": true + }, + "tools": { + "*": { + "run": "ask", + "result": "unattended", + "cancellation_response": "This tool request was intentionally rejected by the user. Please evaluate and either ask the user why it was rejected, or infer the reason by looking at the historical messages in the conversation.", + "style": { + "hidden": false, + "inline_results": { + "truncate": { + "lines": 10 + } + }, + "results_file_link": "full", + "parameters": "json", + "print_stderr": true + } + } + }, + "compaction": { + "rules": { + "value": [ + { + "keep_first": "1", + "keep_last": "1", + "reasoning": "strip", + "tool_calls": "strip" + } + ], + "strategy": "replace", + "discard_when_merged": false + } + }, + "attachments": { + "value": [], + "strategy": "replace", + "discard_when_merged": false + }, + "labels": { + "value": {}, + "strategy": "replace", + "discard_when_merged": false + }, + "start_local": false + }, + "style": { + "code": { + "color": true, + "line_numbers": false, + "file_link": "osc8", + "copy_link": "off" + }, + "markdown": { + "wrap_width": 80, + "table_max_column_width": 40, + "table_continuation_edge": true, + "theme": "gruvbox-dark", + "hr_style": "line" + }, + "mcp_startup": { + "show": true, + "delay_secs": 4, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "reasoning": { + "display": "full", + "background": 236, + "extend_across_tool_calls": true + }, + "streaming": { + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "lock_wait": { + "show": true, + "delay_secs": 1, + "interval_ms": 100, + "timeout_secs": 10 + }, + "tool_call": { + "show": true, + "progress": { + "show": true, + "delay_secs": 3, + "interval_ms": 100, + "stderr_rows": "auto" + }, + "preparing": { + "show": true, + "delay_secs": 3, + "interval_ms": 100 + } + }, + "typewriter": { + "text_delay": "3ms", + "code_delay": "500us", + "max_latency": "0s" + } + }, + "interrupt": { + "escalation_cooldown_secs": 2, + "streaming": { + "action": "prompt", + "compose_in_editor": false + }, + "tool_call": { + "action": "prompt", + "compose_in_editor": false + } + }, + "editor": { + "envs": [ + "JP_EDITOR", + "VISUAL", + "EDITOR" + ], + "inline": { + "edit_mode": "emacs" + } + }, + "providers": { + "llm": { + "anthropic": { + "auth": [ + "api_key" + ], + "api_key_env": "ANTHROPIC_API_KEY", + "base_url": "https://api.anthropic.com", + "chain_on_max_tokens": true, + "beta_headers": [] + }, + "cerebras": { + "api_key_env": "CEREBRAS_API_KEY", + "base_url": "https://api.cerebras.ai" + }, + "deepseek": { + "api_key_env": "DEEPSEEK_API_KEY", + "base_url": "https://api.deepseek.com" + }, + "google": { + "api_key_env": "GEMINI_API_KEY", + "base_url": "https://generativelanguage.googleapis.com/v1beta" + }, + "llamacpp": { + "base_url": "http://127.0.0.1:8080" + }, + "ollama": { + "base_url": "http://localhost:11434" + }, + "openai": { + "api_key_env": "OPENAI_API_KEY", + "base_url": "https://api.openai.com", + "base_url_env": "OPENAI_BASE_URL" + }, + "openrouter": { + "api_key_env": "OPENROUTER_API_KEY", + "app_name": "JP", + "base_url": "https://openrouter.ai" + }, + "vllm": { + "api_key_env": "VLLM_API_KEY", + "base_url": "http://127.0.0.1:8000" + } + } + }, + "plugins": { + "auto_install": true, + "shutdown_timeout_secs": 5 + } + }, + "events": [ + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_request", + "content": "Please run the tool, providing whatever arguments you want." + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "I'll call the tool with some values that fit its schema:\n\n" + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_request", + "id": "chatcmpl-tool-8bef0cd026442499", + "name": "run_me", + "arguments": { + "bar": [ + "Zm9v", + "YmFy" + ], + "foo": "SGVsbG8sIHRvb2wh" + } + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "tool_call_response", + "id": "chatcmpl-tool-8bef0cd026442499", + "content": "d29ya2luZyE=", + "is_error": false + }, + { + "timestamp": "2020-01-01 00:00:00.0", + "type": "chat_response", + "message": "The tool ran successfully and returned **\"working!\"**" + } + ] +} diff --git a/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream__raw_events.snap b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream__raw_events.snap new file mode 100644 index 000000000..1d9d8e187 --- /dev/null +++ b/crates/jp_llm/tests/fixtures/vllm/test_tool_call_stream__raw_events.snap @@ -0,0 +1,150 @@ +--- +source: crates/jp_test/src/mock.rs +expression: v +--- +[ + [ + Part { + index: 1, + part: Message( + "I'", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ll call the tool wi", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "th some", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + " valu", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "es that ", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "fit its sc", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "hema:\n\n", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + Start { + id: "chatcmpl-tool-8bef0cd026442499", + name: "run_me", + }, + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "{\"bar\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "[\"foo\", \"bar\"], \"foo\": ", + ), + ), + metadata: {}, + }, + Part { + index: 2, + part: ToolCall( + ArgumentChunk( + "\"Hello, tool!\"}", + ), + ), + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Flush { + index: 2, + metadata: {}, + }, + Finished( + Completed, + ), + ], + [ + Part { + index: 1, + part: Message( + "The tool ran successfu", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "lly and retur", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ned **\"work", + ), + metadata: {}, + }, + Part { + index: 1, + part: Message( + "ing!\"**", + ), + metadata: {}, + }, + Flush { + index: 0, + metadata: {}, + }, + Flush { + index: 1, + metadata: {}, + }, + Finished( + Completed, + ), + ], +] diff --git a/docs/features/tools.md b/docs/features/tools.md index 4a8371e76..7adfe598d 100644 --- a/docs/features/tools.md +++ b/docs/features/tools.md @@ -197,7 +197,7 @@ Describe the parameter in configuration when that happens. Providers accept different subsets of JSON Schema, and each one adapts the schema itself. -OpenAI, Google, Anthropic, Cerebras, OpenRouter, and llama.cpp all accept +OpenAI, Google, Anthropic, Cerebras, OpenRouter, llama.cpp, and vLLM all accept references and definitions. Ollama does not, so its schemas are expanded before the request is sent. It also ignores keywords outside a small set, keeping `type`, `description`, diff --git a/docs/ticket/063sw3q-exercise-recorded-provider-tool-rounds-through-a-production.md b/docs/ticket/063sw3q-exercise-recorded-provider-tool-rounds-through-a-production.md index 90b25a360..a72a5b8c7 100644 --- a/docs/ticket/063sw3q-exercise-recorded-provider-tool-rounds-through-a-production.md +++ b/docs/ticket/063sw3q-exercise-recorded-provider-tool-rounds-through-a-production.md @@ -28,3 +28,54 @@ Acceptance criteria: - Assert that forced choice becomes `Auto` while tools remain declared. - Keep lower-level provider serialization tests where useful, but name them as such. + +## Comments + +----- + +- **From**: jp +- **Date**: 2026-09-21T19:55:44Z + +The same construction drops reasoning, not just tools. + +`test.rs:591` builds the follow-up as a fresh `TestRequest::chat(provider_id)`, +which seeds `reasoning = Off` alongside the empty tool list. +So `tool_call_reasoning` and `tool_call_required_reasoning` enable reasoning on +the first request and silently disable it on the second. + +The vLLM fixture shows what that produces. +The follow-up replays the prior reasoning in the history while telling the +server not to think: + +```yaml +{ + "role": "assistant", + "reasoning_content": "The user wants me to run the tool with whatever arguments...", + "tool_calls": [ ... ] +}, +... +"chat_template_kwargs": { "enable_thinking": false } +``` + +llama.cpp's fixture carries the same `true` then `false` pair, so this is every +provider, not a vLLM quirk. + +The consequence is that no test covers sending prior reasoning back to a +provider with reasoning still on. +That is the fragile path: Anthropic requires thinking blocks replayed with their +signatures, and `openai_tests.rs` carries a dedicated recorded test for replayed +native reasoning items because the provider rejects a malformed replay. +The shared suite names two tests after reasoning and exercises neither +round-trip. + +Worth folding into this ticket rather than filing separately: it is the same +line, the same fix shape, and the same re-recording cost across seven live +endpoints. +Splitting them means recording every provider twice. + +Suggested additional acceptance criteria: + +- Carry the first request's reasoning setting into the post-result request, so a + reasoning test stays a reasoning test for the whole exchange. +- Assert the replayed assistant turn keeps whatever the provider needs to accept + it (Anthropic's thinking signature, OpenAI's native reasoning item id). diff --git a/docs/ticket/0khs4b1-add-a-vllm-provider.md b/docs/ticket/0khs4b1-add-a-vllm-provider.md new file mode 100644 index 000000000..308803fff --- /dev/null +++ b/docs/ticket/0khs4b1-add-a-vllm-provider.md @@ -0,0 +1,76 @@ +# Add a vLLM provider + +- **Status**: Done +- **Kind**: Feature +- **Authors**: jp +- **Date**: 2026-09-16 +- **Label**: domain=llm +- **Label**: package=jp_config +- **Label**: package=jp_llm +- **Label**: type=feature + +vLLM serves `GET /v1/models` and `POST /v1/chat/completions` behind a Bearer +token, speaking the same Chat Completions dialect that +`jp_llm::provider::openai_compat` already parses for llama.cpp. +A vLLM provider is therefore mostly config plumbing plus a thin provider module +over the shared dialect code. + +## Config + +- `ProviderId::Vllm` in `jp_config::model::id`, with `as_str() == "vllm"`. +- `providers/llm/vllm.rs` holding `VllmConfig`: `api_key_env` defaulting to + `VLLM_API_KEY`, and `base_url` defaulting to `http://127.0.0.1:8000`. + The four trait impls (`AssignKeyValue`, `PartialConfigDelta`, `FillDefaults`, + `ToPartial`) follow `deepseek.rs`. +- A `vllm` field on `LlmProviderConfig`, wired into each of the four impls in + `providers/llm.rs`. +- The `jp_config` snapshots for config fields, schema shape, and partial + defaults all move; review them with `cargo insta`. + +## Provider + +- Hoist `to_system_messages`, `convert_events`, `convert_tools`, and + `convert_tool_choice` out of `llamacpp.rs` into `openai_compat.rs` as + `pub(crate)`, and have llamacpp call them there. + Behavior-preserving. +- `provider/vllm.rs` with `Vllm { client, base_url }`. + `TryFrom<&VllmConfig>` reads the key from the environment and sets the Bearer + header, as `cerebras.rs` does. +- `models()` maps `GET /v1/models` entries to `ModelDetails`, taking + `context_window` from the reported `max_model_len` and keeping the full id + (e.g. `Qwen/Qwen3-8B`) as the name. + `model_details()` returns `ModelDetails::empty()` for an unknown name. +- `create_request()` builds the chat body: model, messages, stream, temperature, + top_p, max_tokens, tools, tool_choice, `response_format` for a structured + schema, and `chat_template_kwargs.enable_thinking` from the reasoning setting. + No `reasoning_format` field — that one is llama.cpp-specific. + `chat_completion_stream()` posts it and parses the SSE stream with + `parse_chunk()`. +- `mod vllm`, the `get_provider()` and `build_request_value()` arms in + `provider.rs`, and the `Vllm` arms in `test.rs` for `base_url` and + `api_key_env`. +- `vllm_tests.rs` covering `create_request()` for a plain message, a tool call + round trip, a structured schema, and reasoning off, each against a static + expected JSON body. + +## Docs + +Name vLLM in the provider sentence in `docs/features/tools.md`. + +## Comments + +----- + +- **From**: jp +- **Date**: 2026-09-16T02:01:11Z + +Filed after the fact: the config, provider, and docs work described above is +implemented and staged. +Deviations from the original plan worth recording: + +- The plan also called for adding vLLM to a provider list in + `docs/configuration.md`. + That file has no provider list, so there is nothing to add there. +- The plan's dotfiles step (the `nebius` alias and the `[providers.llm.vllm]` + table in `agentic-shepherd/dotfiles/jp-user-config/user-config.toml`) lives in + another repository and is out of scope here.