From a6d8f8ae8c403d53aab2ff6280cb696b07a9bafc Mon Sep 17 00:00:00 2001 From: Jean Mertz Date: Wed, 9 Sep 2026 13:59:55 +0200 Subject: [PATCH] fix(llm, cli): Stop OpenAI silently shortening summary requests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summarizing a turn range larger than an OpenAI model's context window now fails with the API's rejection instead of succeeding with a summary that only covers part of the range. `jp conversation compact --turn=2..150 --summary` against a GPT model previously returned a normal completed response built from whatever OpenAI kept, and that summary was stored as standing for all 149 turns; every later request projected it in place of turns the model never read. The cause is `truncation: auto`, which JP sent on every OpenAI request. It lets the API drop input items from the middle of a conversation and answer anyway, so the response is indistinguishable from one that read everything. `summarize_events` rejects a response the *model* cut short, but a request the *provider* cut short arrives as an ordinary `Completed` and passed straight through. `ChatQuery` gains a `truncation` field carrying whether the provider may drop input. Summarization asks for `Forbidden` and gets a 400 it can report; the main query loop and tool inquiries keep `Allowed`, since nothing fits the stream to the window on the query path and a long conversation stays answerable only if the provider may drop what it cannot hold. Only OpenAI offers the choice — the other providers either always reject an oversized request or truncate server-side, out of JP's reach. Signed-off-by: Jean Mertz --- .../jp_cli/src/cmd/conversation/summarize.rs | 11 ++++ crates/jp_cli/src/cmd/query/tool/inquiry.rs | 3 +- crates/jp_cli/src/cmd/query/turn_loop.rs | 6 +- crates/jp_llm/src/provider/anthropic.rs | 1 + crates/jp_llm/src/provider/anthropic_tests.rs | 33 +++++++++- crates/jp_llm/src/provider/cerebras.rs | 1 + crates/jp_llm/src/provider/cerebras_tests.rs | 5 +- crates/jp_llm/src/provider/google.rs | 1 + crates/jp_llm/src/provider/google_tests.rs | 12 +++- crates/jp_llm/src/provider/llamacpp.rs | 1 + crates/jp_llm/src/provider/llamacpp_tests.rs | 3 +- crates/jp_llm/src/provider/mock_tests.rs | 2 + crates/jp_llm/src/provider/ollama.rs | 1 + crates/jp_llm/src/provider/ollama_tests.rs | 3 +- crates/jp_llm/src/provider/openai.rs | 11 +++- crates/jp_llm/src/provider/openai_tests.rs | 62 ++++++++++++++++++- crates/jp_llm/src/provider/openrouter.rs | 1 + .../jp_llm/src/provider/openrouter_tests.rs | 3 +- crates/jp_llm/src/query.rs | 26 ++++++++ crates/jp_llm/src/retry_tests.rs | 2 + crates/jp_llm/src/test.rs | 3 +- crates/jp_llm/src/title.rs | 3 +- 22 files changed, 180 insertions(+), 14 deletions(-) diff --git a/crates/jp_cli/src/cmd/conversation/summarize.rs b/crates/jp_cli/src/cmd/conversation/summarize.rs index 9ddbd5941..c009c76f9 100644 --- a/crates/jp_cli/src/cmd/conversation/summarize.rs +++ b/crates/jp_cli/src/cmd/conversation/summarize.rs @@ -15,6 +15,7 @@ use jp_llm::{ event_builder::EventBuilder, model::ModelDetails, provider, + query::Truncation, retry::{RetryConfig, collect_with_retry}, window, }; @@ -184,6 +185,10 @@ async fn summarize_stream( }, tools: vec![], tool_choice: jp_config::assistant::tool_choice::ToolChoice::default(), + // The summary is stored as standing for every turn in the range, + // so a provider that quietly dropped part of the request would + // hand back a summary covering less than it claims. + truncation: Truncation::Forbidden, }; let llm_events = collect_with_retry(provider, model_details, query, &retry_config).await?; @@ -241,6 +246,12 @@ enum StreamOutcome { /// truncated or declined response would otherwise be stored as the summary and /// replace the turns it was meant to stand in for, silently dropping whatever /// the model never got to. +/// +/// This covers a response the model cut short. +/// A request the *provider* cut short arrives here as an ordinary +/// [`FinishReason::Completed`] and is indistinguishable from a complete one, so +/// that half is prevented when the request is built, by asking for +/// [`Truncation::Forbidden`]. fn summarize_events(events: Vec) -> StreamOutcome { let mut builder = EventBuilder::new(); let mut flushed = Vec::new(); diff --git a/crates/jp_cli/src/cmd/query/tool/inquiry.rs b/crates/jp_cli/src/cmd/query/tool/inquiry.rs index d11434ba5..135810679 100644 --- a/crates/jp_cli/src/cmd/query/tool/inquiry.rs +++ b/crates/jp_cli/src/cmd/query/tool/inquiry.rs @@ -29,7 +29,7 @@ use jp_llm::{ Provider, event_builder::structured_data, model::ModelDetails, - query::ChatQuery, + query::{ChatQuery, Truncation}, retry::{RetryConfig, collect_with_retry}, tool::ToolDefinition, window, @@ -310,6 +310,7 @@ impl InquiryBackend for LlmInquiryBackend { thread, tools: self.tools.clone(), tool_choice: ToolChoice::None, + truncation: Truncation::Allowed, }; let retry_config = diff --git a/crates/jp_cli/src/cmd/query/turn_loop.rs b/crates/jp_cli/src/cmd/query/turn_loop.rs index 215e0ddf4..dfafa3781 100644 --- a/crates/jp_cli/src/cmd/query/turn_loop.rs +++ b/crates/jp_cli/src/cmd/query/turn_loop.rs @@ -35,7 +35,7 @@ use jp_llm::{ event::{Event, EventPart, FinishReason, ToolCallPart}, model::ModelDetails, provider::get_provider, - query::ChatQuery, + query::{ChatQuery, Truncation}, tool::{InvocationContext, ToolDefinition, executor::Executor}, with_idle_timeout, with_output_limit, }; @@ -310,6 +310,10 @@ pub(super) async fn run_turn_loop( thread, tools: tools.to_vec(), tool_choice: tool_choice.clone(), + // Nothing fits the stream to the window on this path, so a + // conversation that outgrows it stays answerable only if + // the provider is allowed to drop what it cannot hold. + truncation: Truncation::Allowed, }; // Claim the waiting region BEFORE the HTTP request. Dropping diff --git a/crates/jp_llm/src/provider/anthropic.rs b/crates/jp_llm/src/provider/anthropic.rs index c55abbe5d..e08197ae2 100644 --- a/crates/jp_llm/src/provider/anthropic.rs +++ b/crates/jp_llm/src/provider/anthropic.rs @@ -1213,6 +1213,7 @@ fn create_request( thread, tools, mut tool_choice, + .. } = query; let mut builder = types::CreateMessagesRequestBuilder::default(); diff --git a/crates/jp_llm/src/provider/anthropic_tests.rs b/crates/jp_llm/src/provider/anthropic_tests.rs index a89f705f6..eafc70a18 100644 --- a/crates/jp_llm/src/provider/anthropic_tests.rs +++ b/crates/jp_llm/src/provider/anthropic_tests.rs @@ -15,7 +15,10 @@ use serde_json::Map; use test_log::test; use super::*; -use crate::test::{TestRequest, run_test}; +use crate::{ + query::Truncation, + test::{TestRequest, run_test}, +}; const MAGIC_STRING: &str = "ANTHROPIC_MAGIC_STRING_TRIGGER_REDACTED_THINKING_46C9A13E193C177646C7398A98432ECCCE4C1253D5E2D82641AC0E52CC2876CB"; @@ -278,6 +281,7 @@ fn test_opus_4_6_request_uses_adaptive_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -334,6 +338,7 @@ fn test_opus_4_7_xhigh_effort_mapping() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -385,6 +390,7 @@ fn test_opus_4_6_xhigh_falls_back_to_high() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -429,6 +435,7 @@ fn test_opus_4_6_max_effort_mapping() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -734,6 +741,7 @@ fn test_unknown_model_requests_summarized_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -811,6 +819,7 @@ fn test_unknown_reasoning_infers_adaptive_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -848,6 +857,7 @@ fn tier_request( }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; create_request(&model, query, true, &BetaFeatures(vec![])).map(|(request, ..)| request) @@ -953,6 +963,7 @@ fn test_off_on_unknown_model_attempts_disable() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1070,6 +1081,7 @@ fn test_fable_5_reasoning_off_omits_disabled_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1105,6 +1117,7 @@ fn test_opus_4_5_uses_budgetted_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1157,6 +1170,7 @@ fn test_structured_output_sets_format() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1231,6 +1245,7 @@ fn test_schema_ignored_when_last_event_is_not_chat_request() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1271,6 +1286,7 @@ fn test_adaptive_thinking_with_structured_output() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1333,6 +1349,7 @@ fn test_forced_tool_with_reasoning_returns_fallback() { parameters: json!({ "type": "object", "properties": {} }), }], tool_choice: ToolChoice::Function("my_tool".into()), + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1405,6 +1422,7 @@ fn test_forced_tool_thinking_always_on_uses_escalating_nudge() { parameters: json!({ "type": "object", "properties": {} }), }], tool_choice: ToolChoice::Function("my_tool".into()), + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1489,6 +1507,7 @@ fn test_forced_tool_thinking_always_on_reasoning_off_still_soft_forces() { parameters: json!({ "type": "object", "properties": {} }), }], tool_choice: ToolChoice::Function("my_tool".into()), + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1554,6 +1573,7 @@ fn test_forced_tool_function_multi_tool_preserves_name() { }, ], tool_choice: ToolChoice::Function("commit".into()), + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1621,6 +1641,7 @@ fn test_forced_tool_without_reasoning_no_fallback() { parameters: json!({ "type": "object", "properties": {} }), }], tool_choice: ToolChoice::Required, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1660,6 +1681,7 @@ fn test_auto_tool_choice_with_reasoning_no_fallback() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1802,6 +1824,7 @@ fn test_continue_injected_when_prefill_unsupported() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1851,6 +1874,7 @@ fn test_prefill_preserved_for_supported_models() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1890,6 +1914,7 @@ fn test_no_injection_when_last_message_is_user() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1933,6 +1958,7 @@ fn test_create_request_resends_signed_thinking_as_native_block() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -1989,6 +2015,7 @@ fn test_create_request_resends_redacted_thinking_as_native_block() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -2041,6 +2068,7 @@ fn test_create_request_falls_back_to_think_tags_without_signature() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -2099,6 +2127,7 @@ fn test_create_request_drops_empty_reasoning_instead_of_empty_think_tags() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -2155,6 +2184,7 @@ fn test_create_request_downgrades_trailing_assistant_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); @@ -2220,6 +2250,7 @@ fn test_create_request_drops_trailing_redacted_thinking() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let beta = BetaFeatures(vec![]); diff --git a/crates/jp_llm/src/provider/cerebras.rs b/crates/jp_llm/src/provider/cerebras.rs index eb00cf9ce..40f087d67 100644 --- a/crates/jp_llm/src/provider/cerebras.rs +++ b/crates/jp_llm/src/provider/cerebras.rs @@ -446,6 +446,7 @@ fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(Value, bool thread, tools, tool_choice, + .. } = query; let structured_schema = thread.events.schema(); diff --git a/crates/jp_llm/src/provider/cerebras_tests.rs b/crates/jp_llm/src/provider/cerebras_tests.rs index 22f7f1af5..3cfdb7bec 100644 --- a/crates/jp_llm/src/provider/cerebras_tests.rs +++ b/crates/jp_llm/src/provider/cerebras_tests.rs @@ -4,7 +4,7 @@ use jp_conversation::{ConversationEvent, event::ToolCallRequest}; use reqwest_eventsource::Error as SseError; use super::*; -use crate::provider::openai_compat::StreamChunk; +use crate::{provider::openai_compat::StreamChunk, query::Truncation}; /// Regression: a model absent from the table must still request the parsed /// reasoning format. @@ -25,6 +25,7 @@ fn test_unknown_model_requests_parsed_reasoning() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let (body, _) = create_request(&model, query).unwrap(); @@ -51,6 +52,7 @@ fn reasoning_query(reasoning: jp_config::model::parameters::PartialReasoningConf }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), } } @@ -70,6 +72,7 @@ fn tier_query(tier: Option) -> ChatQuery { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), } } diff --git a/crates/jp_llm/src/provider/google.rs b/crates/jp_llm/src/provider/google.rs index 4b31a84b7..12562a506 100644 --- a/crates/jp_llm/src/provider/google.rs +++ b/crates/jp_llm/src/provider/google.rs @@ -191,6 +191,7 @@ fn create_request( thread, tools, tool_choice, + .. } = query; // Extract schema and config before into_parts() consumes the thread. diff --git a/crates/jp_llm/src/provider/google_tests.rs b/crates/jp_llm/src/provider/google_tests.rs index 995722190..e18136342 100644 --- a/crates/jp_llm/src/provider/google_tests.rs +++ b/crates/jp_llm/src/provider/google_tests.rs @@ -6,7 +6,10 @@ use jp_test::function_name; use test_log::test; use super::*; -use crate::test::{TestRequest, run_test}; +use crate::{ + query::Truncation, + test::{TestRequest, run_test}, +}; // TODO: Test specific conditions as detailed in // : @@ -53,6 +56,7 @@ fn test_unknown_model_requests_thoughts() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let (request, _) = create_request(&model, query).unwrap(); @@ -188,6 +192,7 @@ fn test_off_on_unknown_model_attempts_disable() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let (request, _) = create_request(&model, query).unwrap(); @@ -227,6 +232,7 @@ fn test_off_on_always_on_leveled_model_uses_lowest_level() { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let (request, _) = create_request(&model, query).unwrap(); @@ -1269,7 +1275,7 @@ mod service_tier_configuration { use crate::{ model::ModelDetails, provider::{Google, ProviderId}, - query::ChatQuery, + query::{ChatQuery, Truncation}, }; static PROVIDER: ProviderId = ProviderId::Google; @@ -1304,6 +1310,7 @@ mod service_tier_configuration { }, tools: vec![], tool_choice: jp_config::assistant::tool_choice::ToolChoice::Auto, + truncation: Truncation::default(), } } @@ -1367,6 +1374,7 @@ mod service_tier_configuration { }, tools: vec![], tool_choice: jp_config::assistant::tool_choice::ToolChoice::Auto, + truncation: Truncation::default(), }; let model = ModelDetails::empty((PROVIDER, "gemini-2.5-flash").try_into().unwrap()); diff --git a/crates/jp_llm/src/provider/llamacpp.rs b/crates/jp_llm/src/provider/llamacpp.rs index 35e244718..b7169f0c9 100644 --- a/crates/jp_llm/src/provider/llamacpp.rs +++ b/crates/jp_llm/src/provider/llamacpp.rs @@ -428,6 +428,7 @@ fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(Value, bool thread, tools, tool_choice, + .. } = query; let structured_schema = thread.events.schema(); diff --git a/crates/jp_llm/src/provider/llamacpp_tests.rs b/crates/jp_llm/src/provider/llamacpp_tests.rs index c6921ced0..7dbe45f66 100644 --- a/crates/jp_llm/src/provider/llamacpp_tests.rs +++ b/crates/jp_llm/src/provider/llamacpp_tests.rs @@ -4,7 +4,7 @@ use jp_conversation::ConversationEvent; use reqwest_eventsource::Error as SseError; use super::*; -use crate::provider::openai_compat::StreamChunk; +use crate::{provider::openai_compat::StreamChunk, query::Truncation}; fn qwen_model() -> LlamacppModel { serde_json::from_value(serde_json::json!({ @@ -70,6 +70,7 @@ fn reasoning_query( }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), } } diff --git a/crates/jp_llm/src/provider/mock_tests.rs b/crates/jp_llm/src/provider/mock_tests.rs index 176b70b9c..35f1fcb20 100644 --- a/crates/jp_llm/src/provider/mock_tests.rs +++ b/crates/jp_llm/src/provider/mock_tests.rs @@ -2,6 +2,7 @@ use futures::StreamExt; use jp_conversation::{ConversationStream, thread::Thread}; use super::*; +use crate::query::Truncation; fn empty_query() -> ChatQuery { ChatQuery { @@ -13,6 +14,7 @@ fn empty_query() -> ChatQuery { }, tools: vec![], tool_choice: jp_config::assistant::tool_choice::ToolChoice::Auto, + truncation: Truncation::default(), } } diff --git a/crates/jp_llm/src/provider/ollama.rs b/crates/jp_llm/src/provider/ollama.rs index f0738e30d..5c08e519a 100644 --- a/crates/jp_llm/src/provider/ollama.rs +++ b/crates/jp_llm/src/provider/ollama.rs @@ -259,6 +259,7 @@ fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(ChatMessage thread, tools, tool_choice, + .. } = query; let structured_schema = thread diff --git a/crates/jp_llm/src/provider/ollama_tests.rs b/crates/jp_llm/src/provider/ollama_tests.rs index 55f0cc375..74154c59b 100644 --- a/crates/jp_llm/src/provider/ollama_tests.rs +++ b/crates/jp_llm/src/provider/ollama_tests.rs @@ -1,7 +1,7 @@ use serde_json::json; use super::*; -use crate::tool::ToolDocs; +use crate::{query::Truncation, tool::ToolDocs}; /// Ollama drops `$ref` while decoding a tool's parameters, so a referenced type /// has to arrive expanded or the model sees a property with no type. @@ -80,6 +80,7 @@ fn reasoning_query( }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), } } diff --git a/crates/jp_llm/src/provider/openai.rs b/crates/jp_llm/src/provider/openai.rs index e9e7409f3..e42de6e74 100644 --- a/crates/jp_llm/src/provider/openai.rs +++ b/crates/jp_llm/src/provider/openai.rs @@ -36,7 +36,7 @@ use crate::{ event::{Event, FinishReason}, model::{ModelDeprecation, ReasoningDetails}, provider::trace_to_tmpfile, - query::ChatQuery, + query::{ChatQuery, Truncation}, stream::with_tool_call_keepalive, tool::{ToolDefinition, json_schema}, }; @@ -353,6 +353,7 @@ fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(Request, bo thread, tools, tool_choice, + truncation, } = query; let config = thread.events.config()?; @@ -582,7 +583,13 @@ fn create_request(model: &ModelDetails, query: ChatQuery) -> Result<(Request, bo temperature, reasoning, max_output_tokens: parameters.max_tokens.map(Into::into), - truncation: Some(types::Truncation::Auto), + // `Auto` drops input items from the middle of the conversation and + // answers anyway, so a caller that needs the whole input read asks for + // `Forbidden` and gets a 400 instead of a quietly partial answer. + truncation: Some(match truncation { + Truncation::Allowed => types::Truncation::Auto, + Truncation::Forbidden => types::Truncation::Disabled, + }), top_p, text, // OpenAI routes requests by prompt prefix; a stable per-conversation diff --git a/crates/jp_llm/src/provider/openai_tests.rs b/crates/jp_llm/src/provider/openai_tests.rs index 3ac6c7b8d..0735a0e60 100644 --- a/crates/jp_llm/src/provider/openai_tests.rs +++ b/crates/jp_llm/src/provider/openai_tests.rs @@ -6,7 +6,11 @@ mod service_tier { use serde_json::json; use super::super::create_request; - use crate::{model::ModelDetails, provider::ProviderId, query::ChatQuery}; + use crate::{ + model::ModelDetails, + provider::ProviderId, + query::{ChatQuery, Truncation}, + }; fn request_tier(tier: Option) -> Option { let mut events = ConversationStream::new_test().with_turn("test"); @@ -23,6 +27,7 @@ mod service_tier { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), }; let model = ModelDetails::empty((ProviderId::Openai, "gpt-5.6").try_into().unwrap()); @@ -63,6 +68,61 @@ mod service_tier { } } +mod truncation { + use jp_config::assistant::tool_choice::ToolChoice; + use jp_conversation::{ConversationStream, thread::Thread}; + use serde_json::json; + + use super::super::create_request; + use crate::{ + model::ModelDetails, + provider::ProviderId, + query::{ChatQuery, Truncation}, + }; + + /// The value the built request sends in its `truncation` field. + fn request_truncation(truncation: Truncation) -> Option { + let query = ChatQuery { + thread: Thread { + system_prompt: None, + sections: vec![], + attachments: vec![], + events: ConversationStream::new_test().with_turn("test"), + }, + tools: vec![], + tool_choice: ToolChoice::Auto, + truncation, + }; + + let model = ModelDetails::empty((ProviderId::Openai, "gpt-5.6").try_into().unwrap()); + let (request, ..) = create_request(&model, query).unwrap(); + + serde_json::to_value(request) + .unwrap() + .get("truncation") + .cloned() + } + + /// A request that may be truncated lets the API drop input to fit, which + /// keeps a long conversation answerable rather than failing it outright. + #[test] + fn an_allowed_request_asks_for_auto() { + assert_eq!(request_truncation(Truncation::Allowed), Some(json!("auto"))); + } + + /// A caller that stores the answer as standing for its input needs the + /// request rejected rather than silently shortened: `auto` drops items from + /// the middle of the conversation and answers anyway, which reads as an + /// ordinary success. + #[test] + fn a_forbidden_request_asks_for_disabled() { + assert_eq!( + request_truncation(Truncation::Forbidden), + Some(json!("disabled")) + ); + } +} + mod make_schema_nullable { use serde_json::json; diff --git a/crates/jp_llm/src/provider/openrouter.rs b/crates/jp_llm/src/provider/openrouter.rs index be47d0ca3..40b938843 100644 --- a/crates/jp_llm/src/provider/openrouter.rs +++ b/crates/jp_llm/src/provider/openrouter.rs @@ -744,6 +744,7 @@ fn create_request( thread, tools, tool_choice, + .. } = query; let config = thread.events.config()?; diff --git a/crates/jp_llm/src/provider/openrouter_tests.rs b/crates/jp_llm/src/provider/openrouter_tests.rs index 3540ae7d9..abb19f470 100644 --- a/crates/jp_llm/src/provider/openrouter_tests.rs +++ b/crates/jp_llm/src/provider/openrouter_tests.rs @@ -4,7 +4,7 @@ use jp_test::{Result, function_name}; use serde_json::json; use super::*; -use crate::{model::ReasoningDetails, test::TestRequest}; +use crate::{model::ReasoningDetails, query::Truncation, test::TestRequest}; macro_rules! test_all_models { ($($fn:ident),* $(,)?) => { @@ -111,6 +111,7 @@ fn tier_query(tier: Option) -> ChatQuery { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), } } diff --git a/crates/jp_llm/src/query.rs b/crates/jp_llm/src/query.rs index b06457702..77957d589 100644 --- a/crates/jp_llm/src/query.rs +++ b/crates/jp_llm/src/query.rs @@ -3,6 +3,25 @@ use jp_conversation::thread::Thread; use crate::tool::ToolDefinition; +/// Whether the provider may drop input to make a request fit the model's +/// context window. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub enum Truncation { + /// The provider may silently discard input and answer anyway. + /// + /// Suits a request whose answer stands on its own, where a degraded answer + /// beats no answer. + #[default] + Allowed, + + /// The provider must reject a request that does not fit. + /// + /// Required when the answer is stored as standing for the input it was + /// built from: a silently shortened request yields an answer that claims + /// coverage it never had. + Forbidden, +} + #[derive(Debug, Clone)] pub struct ChatQuery { pub thread: Thread, @@ -18,6 +37,12 @@ pub struct ChatQuery { // // Same logic applies here, I think? pub tool_choice: ToolChoice, + + /// Whether the provider may drop input to fit its context window. + /// + /// Only providers that offer the choice read this; the rest either always + /// reject an oversized request or truncate server-side beyond JP's control. + pub truncation: Truncation, } impl From for ChatQuery { @@ -26,6 +51,7 @@ impl From for ChatQuery { thread, tools: vec![], tool_choice: ToolChoice::default(), + truncation: Truncation::default(), } } } diff --git a/crates/jp_llm/src/retry_tests.rs b/crates/jp_llm/src/retry_tests.rs index 1ffd2622a..cdc9b241a 100644 --- a/crates/jp_llm/src/retry_tests.rs +++ b/crates/jp_llm/src/retry_tests.rs @@ -10,6 +10,7 @@ use crate::{ event::Event, model::ModelDetails, provider::mock::MockProvider, + query::Truncation, }; fn empty_query() -> ChatQuery { @@ -22,6 +23,7 @@ fn empty_query() -> ChatQuery { }, tools: vec![], tool_choice: ToolChoice::Auto, + truncation: Truncation::default(), } } diff --git a/crates/jp_llm/src/test.rs b/crates/jp_llm/src/test.rs index f83fbe8aa..d11557d39 100644 --- a/crates/jp_llm/src/test.rs +++ b/crates/jp_llm/src/test.rs @@ -26,7 +26,7 @@ use crate::{ event_builder::EventBuilder, model::{ModelDetails, ReasoningDetails}, provider::get_provider, - query::ChatQuery, + query::{ChatQuery, Truncation}, tool::{ToolDefinition, ToolDocs}, }; @@ -225,6 +225,7 @@ impl TestRequest { .unwrap(), tools: vec![], tool_choice: ToolChoice::default(), + truncation: Truncation::default(), }, assert: Arc::new(|_| {}), assert_history: Arc::new(|_| {}), diff --git a/crates/jp_llm/src/title.rs b/crates/jp_llm/src/title.rs index 08cd30087..28b203a35 100644 --- a/crates/jp_llm/src/title.rs +++ b/crates/jp_llm/src/title.rs @@ -28,7 +28,7 @@ use crate::{ error::Result, event_builder, model::ModelDetails, - query::ChatQuery, + query::{ChatQuery, Truncation}, retry::{RetryConfig, collect_with_retry}, window, }; @@ -170,6 +170,7 @@ pub async fn generate( thread, tools: vec![], tool_choice: ToolChoice::default(), + truncation: Truncation::default(), }; let retry = RetryConfig::default().with_max_response_bytes(max_response_bytes);