From 5b83b4fbeaede9fa619d13339f41b7e50ca7012b Mon Sep 17 00:00:00 2001 From: "Julianne H." Date: Fri, 25 Sep 2026 17:06:10 +0200 Subject: [PATCH] feat: add per-request compression toggles --- example/compression.ts | 115 +++++++++++++++++------------------------ src/index.ts | 40 ++++++++++++-- tests/index.test.ts | 87 +++++++++++++++++++++++++++++++ 3 files changed, 172 insertions(+), 70 deletions(-) diff --git a/example/compression.ts b/example/compression.ts index 55b750d..d46388b 100644 --- a/example/compression.ts +++ b/example/compression.ts @@ -2,89 +2,71 @@ * Example: Token compression with Edgee Gateway SDK * * This example demonstrates how to: - * 1. Enable compression for a request with a large input context - * 2. Set a custom compression rate - * 3. Access compression metrics from the response + * 1. Turn tool-result trimming on for a single request + * 2. Access compression metrics from the response * - * IMPORTANT: Only USER messages are compressed. System messages are not compressed. - * This example includes a large context in the user message to demonstrate meaningful - * compression savings. + * Tool-result trimming shortens the output of tool calls (here a long `ls -la` + * listing) before it reaches the model. The per-request toggles + * (`tool_result_trimming`, `tool_surface_reduction`, `output_brevity`) override + * the API key settings for this request only; leave one out to keep the key's + * setting. */ import Edgee from "edgee"; const edgee = new Edgee(process.env.EDGEE_API_KEY); -// Large context document to demonstrate input compression -const LARGE_CONTEXT = ` -The History and Impact of Artificial Intelligence - -Artificial intelligence (AI) has evolved from a theoretical concept to a -transformative technology that influences nearly every aspect of modern life. -The field began in earnest in the 1950s when pioneers like Alan Turing and -John McCarthy laid the groundwork for machine intelligence. - -Early developments focused on symbolic reasoning and expert systems. These -rule-based approaches dominated the field through the 1970s and 1980s, with -systems like MYCIN demonstrating practical applications in medical diagnosis. -However, these early systems were limited by their inability to learn from data -and adapt to new situations. - -The resurgence of neural networks in the 1980s and 1990s, particularly with -backpropagation algorithms, opened new possibilities. Yet it wasn't until the -2010s, with the advent of deep learning and the availability of massive datasets -and computational power, that AI truly began to revolutionize industries. - -Modern AI applications span numerous domains: -- Natural language processing enables machines to understand and generate human language -- Computer vision allows machines to interpret visual information from the world -- Robotics combines AI with mechanical systems for autonomous operation -- Healthcare uses AI for diagnosis, drug discovery, and personalized treatment -- Finance leverages AI for fraud detection, algorithmic trading, and risk assessment -- Transportation is being transformed by autonomous vehicles and traffic optimization - -The development of large language models like GPT, BERT, and others has -particularly accelerated progress in natural language understanding and generation. -These models, trained on vast amounts of text data, can perform a wide range of -language tasks with remarkable proficiency. - -Despite remarkable progress, significant challenges remain. Issues of bias, -interpretability, safety, and ethical considerations continue to be areas of -active research and debate. The AI community is working to ensure that these -powerful technologies are developed and deployed responsibly, with consideration -for their societal impact. - -Looking forward, AI is expected to continue advancing rapidly, with potential -breakthroughs in areas like artificial general intelligence, quantum machine -learning, and brain-computer interfaces. The integration of AI into daily life -will likely deepen, raising important questions about human-AI collaboration, -workforce transformation, and the future of human cognition itself. -`; +// A long directory listing, the kind of tool output coding agents send back. +const LS_OUTPUT = + "total 800\n" + + Array.from( + { length: 200 }, + (_, i) => + `-rw-r--r-- 1 user staff ${1000 + i} Jan 1 12:00 src/components/module_${String(i).padStart(3, "0")}.tsx` + ).join("\n"); console.log("=".repeat(70)); console.log("Edgee Token Compression Example"); console.log("=".repeat(70)); console.log(); -// Example: Request with compression enabled and large input -console.log("Example: Large user message with compression enabled"); +console.log("Example: Large tool result with tool-result trimming turned on"); console.log("-".repeat(70)); -console.log(`Input context length: ${LARGE_CONTEXT.length} characters`); +console.log(`Tool output length: ${LS_OUTPUT.length} characters`); console.log(); -// NOTE: Only USER messages are compressed -// Put the large context in the user message to demonstrate compression -const userMessage = `Here is some context about AI: - -${LARGE_CONTEXT} - -Based on this context, summarize the key milestones in AI development in 3 bullet points.`; - const response = await edgee.send({ model: "anthropic/claude-haiku-4-5", input: { - messages: [{ role: "user", content: userMessage }], - compression_model: "claude", + messages: [ + { role: "user", content: "How many files are in src/components?" }, + { + role: "assistant", + tool_calls: [ + { + id: "call_1", + type: "function", + function: { name: "Bash", arguments: '{"command":"ls -la src/components"}' }, + }, + ], + }, + { role: "tool", tool_call_id: "call_1", content: LS_OUTPUT }, + ], + tools: [ + { + type: "function", + function: { + name: "Bash", + description: "Run a shell command and return its output.", + parameters: { + type: "object", + properties: { command: { type: "string" } }, + required: ["command"], + }, + }, + }, + ], + tool_result_trimming: true, }, }); @@ -125,11 +107,10 @@ if (response.compression) { } else { console.log("No compression data available in response."); console.log( - "Note: Compression data is only returned when compression is enabled" + "Note: Compression data is only returned when trimming actually shortened" ); - console.log(" and supported by your API key configuration."); + console.log(" a tool result."); } console.log(); console.log("=".repeat(70)); - diff --git a/src/index.ts b/src/index.ts index c7c7f7e..6cd46c0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,7 +43,37 @@ export interface InputObject { tools?: Tool[]; tool_choice?: ToolChoice; tags?: string[]; - compression_model?: "claude" | "opencode" | "cursor" | "customer"; // Compression model (gateway-internal, not sent to providers) + /** + * @deprecated Legacy switch: any value turns tool-result trimming on for this + * request. Use `tool_result_trimming` instead. + */ + compression_model?: "claude" | "opencode" | "cursor" | "customer"; + /** Turn tool-result trimming on or off for this request. Omit to keep the API key setting. */ + tool_result_trimming?: boolean; + /** + * Turn MCP tool surface reduction on or off for this request. Omit to keep the + * API key setting. The threshold still comes from the API key. + */ + tool_surface_reduction?: boolean; + /** Turn output brevity on or off for this request. Omit to keep the API key setting. */ + output_brevity?: boolean; +} + +const COMPRESSION_HEADERS = { + tool_result_trimming: "X-Edgee-Compression-Tool-Result-Trimming", + tool_surface_reduction: "X-Edgee-Compression-Tool-Surface-Reduction", + output_brevity: "X-Edgee-Compression-Brevity", +} as const; + +/** Per-request compression overrides, sent only for the toggles the caller set. */ +function compressionHeaders(input: string | InputObject): Record { + const headers: Record = {}; + if (typeof input === "string") return headers; + for (const [field, header] of Object.entries(COMPRESSION_HEADERS)) { + const value = input[field as keyof typeof COMPRESSION_HEADERS]; + if (typeof value === "boolean") headers[header] = String(value); + } + return headers; } export interface SendOptions { @@ -209,6 +239,7 @@ export default class Edgee { headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}`, + ...compressionHeaders(input), }, body: JSON.stringify(body), }); @@ -238,13 +269,15 @@ export default class Edgee { private async *_handleStreamingResponse( url: string, - body: Record + body: Record, + extraHeaders: Record = {} ): AsyncGenerator { const res = await fetch(url, { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}`, + ...extraHeaders, }, body: JSON.stringify(body), }); @@ -314,7 +347,8 @@ export default class Edgee { yield* this._handleStreamingResponse( `${this.baseUrl}/v1/chat/completions`, - body + body, + compressionHeaders(input) ); } } diff --git a/tests/index.test.ts b/tests/index.test.ts index 524dc20..aa6c64b 100644 --- a/tests/index.test.ts +++ b/tests/index.test.ts @@ -589,6 +589,93 @@ describe('Edgee', () => { }) ).rejects.toThrow('API error 500: Internal Server Error'); }); + + describe('compression overrides', () => { + const okResponse = () => ({ + ok: true, + json: async () => ({ + choices: [ + { index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }, + ], + }), + }); + const sentHeaders = () => + (mockFetch.mock.calls[0][1] as { headers: Record }).headers; + const sentBody = () => + JSON.parse((mockFetch.mock.calls[0][1] as { body: string }).body) as Record; + + it('should send each set toggle as its header, never in the body', async () => { + mockFetch.mockResolvedValueOnce(okResponse()); + + await client.send({ + model: 'gpt-4', + input: { + messages: [{ role: 'user', content: 'Hello' }], + tool_result_trimming: true, + tool_surface_reduction: false, + output_brevity: true, + }, + }); + + expect(sentHeaders()).toMatchObject({ + 'X-Edgee-Compression-Tool-Result-Trimming': 'true', + 'X-Edgee-Compression-Tool-Surface-Reduction': 'false', + 'X-Edgee-Compression-Brevity': 'true', + }); + const body = sentBody(); + expect(body).not.toHaveProperty('tool_result_trimming'); + expect(body).not.toHaveProperty('tool_surface_reduction'); + expect(body).not.toHaveProperty('output_brevity'); + }); + + it('should omit the headers for toggles left unset', async () => { + mockFetch.mockResolvedValueOnce(okResponse()); + + await client.send({ + model: 'gpt-4', + input: { messages: [{ role: 'user', content: 'Hello' }], tool_result_trimming: false }, + }); + + const headers = sentHeaders(); + expect(headers['X-Edgee-Compression-Tool-Result-Trimming']).toBe('false'); + expect(headers).not.toHaveProperty('X-Edgee-Compression-Tool-Surface-Reduction'); + expect(headers).not.toHaveProperty('X-Edgee-Compression-Brevity'); + }); + + it('should send no compression headers for string input', async () => { + mockFetch.mockResolvedValueOnce(okResponse()); + + await client.send({ model: 'gpt-4', input: 'Hello' }); + + expect(Object.keys(sentHeaders())).toEqual(['Content-Type', 'Authorization']); + }); + + it('should send the toggles on streaming requests', async () => { + const encoder = new TextEncoder(); + mockFetch.mockResolvedValueOnce({ + ok: true, + body: new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('data: [DONE]\n\n')); + controller.close(); + }, + }), + }); + + const chunks: StreamChunk[] = []; + for await (const chunk of client.stream('gpt-4', { + messages: [{ role: 'user', content: 'Hello' }], + tool_surface_reduction: true, + })) { + chunks.push(chunk); + } + expect(chunks).toHaveLength(0); + + const headers = sentHeaders(); + expect(headers['X-Edgee-Compression-Tool-Surface-Reduction']).toBe('true'); + expect(headers).not.toHaveProperty('X-Edgee-Compression-Tool-Result-Trimming'); + }); + }); }); describe('convenience properties', () => {