chat.js

// Relaying a chat to the AI provider and streaming the reply back.
//
// The browser sends the whole conversation with every message (it lives in the browser's
// localStorage, not here). This file trims it to a sane size, adds the system prompt, calls
// the provider's Chat Completions API with the key, and passes the reply through to the
// browser as server-sent events while it is being generated. The key never leaves the server.

// Limits on what one request may send upstream. Every token of history is paid for again on
// every message, so an unbounded history is an unbounded bill.
const MAX_BODY_BYTES = 128 * 1024;
const MAX_MESSAGES = 40;
const MAX_MESSAGE_CHARS = 8_000;
const MAX_HISTORY_CHARS = 32_000;

const FIRST_TOKEN_TIMEOUT_MS = 60_000; // some models think for a while before the first word
const IDLE_TIMEOUT_MS = 60_000; // silence between chunks after that
const TOTAL_TIMEOUT_MS = 5 * 60_000;
// velixir's edge closes a response that sends nothing for 30 seconds, so a comment goes out
// every 15 while the model is thinking. Browsers ignore SSE comments.
const HEARTBEAT_MS = 15_000;

class RequestError extends Error {
  constructor(status, message) {
    super(message);
    this.status = status;
  }
}

// Keeps the newest messages that fit the limits. Only user and assistant turns are accepted:
// the system prompt is the server's to set, so a client cannot smuggle in its own.
function trimHistory(body) {
  if (!body || !Array.isArray(body.messages)) {
    throw new RequestError(400, 'Expected {"messages": [{"role": "user", "content": "..."}]}.');
  }
  const turns = body.messages
    .filter((m) => m && (m.role === 'user' || m.role === 'assistant') && typeof m.content === 'string')
    .map((m) => ({ role: m.role, content: m.content }));

  const last = turns.at(-1);
  if (!last || last.role !== 'user' || !last.content.trim()) {
    throw new RequestError(400, 'The last message must be yours, and not empty.');
  }
  if (last.content.length > MAX_MESSAGE_CHARS) {
    throw new RequestError(413, `That message is too long. The limit is ${MAX_MESSAGE_CHARS.toLocaleString('en-GB')} characters.`);
  }

  // Walk back from the newest message, keeping whole messages while they fit. An older
  // message that is too long on its own is cut down rather than breaking the conversation.
  const kept = [];
  let total = 0;
  for (let i = turns.length - 1; i >= 0 && kept.length < MAX_MESSAGES; i--) {
    const content = turns[i].content.slice(0, MAX_MESSAGE_CHARS);
    if (total + content.length > MAX_HISTORY_CHARS) break;
    total += content.length;
    kept.unshift({ role: turns[i].role, content });
  }
  // Start on a user turn: some providers reject a conversation that opens with the assistant.
  while (kept.length > 1 && kept[0].role !== 'user') kept.shift();
  return kept;
}

// Anything that looks like a credential is removed before a provider's error message is shown
// in the browser. Some providers quote part of the key back in a 401.
function redact(text, apiKey) {
  let out = String(text ?? '');
  if (apiKey) out = out.split(apiKey).join('[redacted]');
  return out
    .replace(/\b(sk|gsk|pk|rk|key)[-_][A-Za-z0-9*._-]{6,}/gi, '[redacted]')
    .replace(/Bearer\s+\S+/gi, 'Bearer [redacted]')
    .slice(0, 300);
}

// Pulls the human-readable part out of whatever JSON (or text) a provider returns on error.
function providerMessage(text) {
  try {
    const json = JSON.parse(text);
    const e = json.error ?? json;
    const message = typeof e === 'string' ? e : (e.message ?? e.detail ?? json.message ?? json.detail);
    if (message) return typeof message === 'string' ? message : JSON.stringify(message);
  } catch {
    // Not JSON: fall through to the raw text.
  }
  return String(text || '').trim();
}

function describeUpstreamError(status, text, config) {
  const host = new URL(config.baseUrl).host;
  const detail = redact(providerMessage(text), config.apiKey);
  const says = detail ? ` The provider said: ${detail}` : '';

  // No provider text for auth failures: it adds nothing, and it is where key fragments show up.
  if (status === 401 || status === 403) {
    return `The AI provider rejected the API key (${status}). Check that AI_API_KEY is a valid key for ${host}.`;
  }
  if (status === 404) {
    return `The AI provider could not find the model or endpoint (404). Check AI_MODEL ("${config.model}") and AI_BASE_URL.${says}`;
  }
  if (status === 429) {
    return `The AI provider is rate limiting this key, or its credit has run out (429). Wait a moment and try again, or check your plan with the provider.${says}`;
  }
  if (status >= 500) {
    return `The AI provider had a problem on its side (${status}). Try again in a moment.${says}`;
  }
  return `The AI provider refused the request (${status}).${says}`;
}

// The text of one streamed chunk. Usually a string; some providers send a list of parts
// (text and "thinking"), of which only the text is shown.
function deltaText(delta) {
  const content = delta?.content;
  if (typeof content === 'string') return content;
  if (Array.isArray(content)) {
    return content.map((part) => (typeof part === 'string' ? part : part?.type === 'text' ? part.text ?? '' : '')).join('');
  }
  return '';
}

async function streamChat({ config, messages, res }) {
  // Headers go out before the provider is even called. The edge waits at most 30 seconds for
  // them, and a slow provider must not turn into a gateway error. From here on, problems are
  // reported inside the stream as an "error" event.
  res.writeHead(200, {
    'Content-Type': 'text/event-stream; charset=utf-8',
    'Cache-Control': 'no-store, no-transform',
    'X-Accel-Buffering': 'no', // tells nginx-style proxies not to hold the stream back
  });
  res.flushHeaders();

  const send = (event, data) => res.write(`event: ${event}\ndata: ${JSON.stringify(data)}\n\n`);
  const heartbeat = setInterval(() => res.write(': keep-alive\n\n'), HEARTBEAT_MS);

  const controller = new AbortController();
  let stopReason = null;
  const stop = (reason) => {
    if (stopReason) return;
    stopReason = reason;
    controller.abort();
  };
  // The browser went away (tab closed, Stop pressed): stop the provider too, so nobody pays
  // for tokens that no one will read.
  res.on('close', () => stop('client'));
  const total = setTimeout(() => stop('timeout'), TOTAL_TIMEOUT_MS);
  let idle = setTimeout(() => stop('timeout'), FIRST_TOKEN_TIMEOUT_MS);

  const started = Date.now();
  let chars = 0;
  let finishReason = null;
  let failed = false;

  try {
    const upstream = await fetch(`${config.baseUrl}/chat/completions`, {
      method: 'POST',
      headers: {
        Authorization: `Bearer ${config.apiKey}`,
        'Content-Type': 'application/json',
        Accept: 'text/event-stream',
      },
      body: JSON.stringify({
        model: config.model,
        messages: config.systemPrompt ? [{ role: 'system', content: config.systemPrompt }, ...messages] : messages,
        stream: true,
      }),
      signal: controller.signal,
    });

    if (!upstream.ok) {
      const text = (await upstream.text()).slice(0, 4000);
      failed = true;
      console.error(`upstream ${upstream.status}: ${redact(providerMessage(text), config.apiKey)}`);
      send('error', { message: describeUpstreamError(upstream.status, text, config), status: upstream.status });
      return;
    }

    // Server-sent events: blocks separated by a blank line, each with "data:" lines. Lines
    // starting with ":" are comments (OpenRouter sends them as keep-alives) and are skipped.
    const decoder = new TextDecoder();
    let buffer = '';
    let done = false;

    const handleEvent = (block) => {
      const data = block
        .split(/\r?\n/)
        .filter((line) => line.startsWith('data:'))
        .map((line) => line.slice(5).replace(/^ /, ''))
        .join('\n');
      if (!data) return;
      if (data === '[DONE]') {
        done = true;
        return;
      }
      let json;
      try {
        json = JSON.parse(data);
      } catch {
        return; // a malformed chunk is skipped rather than ending the reply
      }
      // Some providers report a failure mid-stream, after the 200 has gone out.
      if (json.error) {
        failed = true;
        done = true;
        const message = redact(providerMessage(JSON.stringify(json)), config.apiKey) || 'unknown error';
        send('error', { message: `The AI provider stopped with an error: ${message}` });
        return;
      }
      const choice = json.choices?.[0];
      const text = deltaText(choice?.delta);
      if (text) {
        chars += text.length;
        send('delta', { text });
      }
      if (choice?.finish_reason) finishReason = choice.finish_reason;
    };

    for await (const chunk of upstream.body) {
      clearTimeout(idle);
      idle = setTimeout(() => stop('timeout'), IDLE_TIMEOUT_MS);
      buffer += decoder.decode(chunk, { stream: true });
      const blocks = buffer.split(/\r?\n\r?\n/);
      buffer = blocks.pop();
      for (const block of blocks) {
        handleEvent(block);
        if (done) break;
      }
      if (done) break;
    }
    if (!done && buffer.trim()) handleEvent(buffer);

    if (!failed) send('done', { finishReason });
  } catch (err) {
    if (stopReason === 'client') return; // nobody left to tell
    failed = true;
    const host = new URL(config.baseUrl).host;
    const message = stopReason === 'timeout'
      ? `The AI provider at ${host} stopped responding, so the request was cancelled. Try again.`
      : `Could not reach the AI provider at ${host} (${err.cause?.code || err.message}). Check AI_BASE_URL.`;
    console.error(`upstream request failed: ${stopReason === 'timeout' ? 'timed out' : err.cause?.code || err.message}`);
    send('error', { message });
  } finally {
    clearInterval(heartbeat);
    clearTimeout(total);
    clearTimeout(idle);
    res.end();
    // Sizes and timings only. The conversation itself is never logged.
    console.log(`chat: ${messages.length} messages in, ${chars} characters out, ${Date.now() - started} ms` +
      `${finishReason ? `, finish ${finishReason}` : ''}${failed ? ', failed' : ''}${stopReason === 'client' ? ', cancelled by the browser' : ''}`);
  }
}

module.exports = { trimHistory, streamChat, RequestError, MAX_BODY_BYTES, MAX_MESSAGE_CHARS };