From 88a6ae2258c62f286273b7d0ccd6902588d9ec71 Mon Sep 17 00:00:00 2001 From: YUZHEthefool <2804776511@qq.com> Date: Tue, 25 Aug 2026 17:34:31 +0800 Subject: [PATCH] feat(retry): auto-retry Cloudflare 5xx from relays + user-configurable retry-error settings (#608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relays/proxy stations intermittently surface Cloudflare 520/521/525 and similar transient 5xx. pi-ai's isRetryableAssistantError only covers 429/500/502/503/504/524, so these errors failed outright instead of retrying (#608). This layers a LiveAgent retry-error extension on top of pi-ai's classifier, applied to both withStreamRetry and withProviderFailover so a transient relay 5xx now retries with exponential backoff (matching 524) and can also trigger a provider switch. - Default the extension to every Cloudflare preset (520-527) on, so relays self-heal out of the box with zero configuration. - Add a global RetryErrorSettings (preset status-code toggles + custom substrings) persisted in localStorage. Surfaced as a card inside the Providers custom-settings drawer (alongside failover), per maintainer review, rather than a standalone settings page. - Sync settings into the runtime via a module-level extension, so all streaming paths (agent runs, text mode, titles, compaction, memory, cron, subagents) pick up the classification without per-call plumbing. - Preserve pi-ai's non-retryable quota/billing guard and the failover ineligible client-error guard as hard floors. Tests: - stream-retry: extension classifier (preset code match, word-boundary guard so "520" != "5200", case-insensitive substring, empty/undefined message, multi-code alternation, whitespace-only pattern dropped), withStreamRetry retrying 525/custom patterns via retryExtension and the module default (zero-config fix), preset-disabled not retried, quota error still not retried, failover eligibility (+ client-error still rejected), setRetryErrorExtension/getRetryErrorExtension round-trip. - retry-error-normalize: defaults, preset validation/dedup, empty-array opt-out, legacy fallback, custom-pattern trim/dedup, normalizeSettings wiring. - storage: retryErrorSettings localStorage round-trip — defaults when empty, read+normalize on load, legacy fallback to all presets. Closes #608. --- crates/agent-gui/src/App.tsx | 13 ++ .../lib/providers/runtime/providerFailover.ts | 19 +- .../src/lib/providers/runtime/streamRetry.ts | 93 +++++++- crates/agent-gui/src/lib/settings/storage.ts | 17 +- .../test/providers/stream-retry.test.mjs | 204 +++++++++++++++++- .../settings/retry-error-normalize.test.mjs | 57 +++++ .../agent-gui/test/settings/storage.test.mjs | 74 +++++++ .../src/i18n/translations/enUSSettings.ts | 20 ++ .../src/i18n/translations/zhCNSettings.ts | 20 ++ crates/agent-ui/src/lib/settings/index.ts | 6 + .../agent-ui/src/lib/settings/retryError.ts | 57 +++++ crates/agent-ui/src/lib/settings/types.ts | 36 ++++ .../src/pages/settings/ProvidersSection.tsx | 2 + .../src/pages/settings/RetryErrorSection.tsx | 174 +++++++++++++++ 14 files changed, 787 insertions(+), 5 deletions(-) create mode 100644 crates/agent-gui/test/settings/retry-error-normalize.test.mjs create mode 100644 crates/agent-ui/src/lib/settings/retryError.ts create mode 100644 crates/agent-ui/src/pages/settings/RetryErrorSection.tsx diff --git a/crates/agent-gui/src/App.tsx b/crates/agent-gui/src/App.tsx index 402b49b03..ab4be69e1 100644 --- a/crates/agent-gui/src/App.tsx +++ b/crates/agent-gui/src/App.tsx @@ -27,6 +27,7 @@ import { AppBootShell } from "./components/app/AppBootShell"; import { useNativeInputContextMenu } from "./components/input-context-menu/NativeInputContextMenu"; import { WindowsTitleBar } from "./components/WindowsTitleBar"; import { useAppUpdateController } from "./lib/appUpdates"; +import { setRetryErrorExtension } from "./lib/providers/runtime/streamRetry"; import { type AppSettings, getDefaultSettings, @@ -387,6 +388,18 @@ export default function App() { return () => window.clearTimeout(timeoutId); }, [settingsReady]); + // Push the user's retry-error classification (preset Cloudflare 5xx toggles + + // custom substrings) into the stream-retry runtime. The extension is a pure + // function of settings, so re-running on every change keeps the runtime in + // sync without any per-call plumbing. The runtime's default already enables + // every preset, so this is a no-op until the user actually changes something. + useEffect(() => { + setRetryErrorExtension({ + statusCodes: settings.retryErrorSettings.presetStatusCodes, + patterns: settings.retryErrorSettings.customPatterns, + }); + }, [settings.retryErrorSettings]); + const queueSettingsSave = useCallback( (prev: AppSettings, next: AppSettings, fallback: string, publishSync: boolean) => { const saveSequence = ++saveSequenceRef.current; diff --git a/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts b/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts index 958453832..9e935df51 100644 --- a/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts +++ b/crates/agent-gui/src/lib/providers/runtime/providerFailover.ts @@ -5,6 +5,7 @@ import { createAssistantMessageEventStream, isRetryableAssistantError, } from "@earendil-works/pi-ai"; +import { isExtensionRetryableError, type RetryErrorExtension } from "./streamRetry"; /** * Provider auto-failover runtime (cc-switch inspired). @@ -182,11 +183,19 @@ const FAILOVER_EXTRA_ELIGIBLE_ERROR_PATTERN = new RegExp( * style client errors never fail over even though they may contain digits * that look like status codes. */ -export function isFailoverEligibleAssistantError(message: AssistantMessage | undefined): boolean { +export function isFailoverEligibleAssistantError( + message: AssistantMessage | undefined, + retryExtension?: RetryErrorExtension, +): boolean { if (!message) return false; const errorMessage = (message as { errorMessage?: string }).errorMessage ?? ""; if (FAILOVER_INELIGIBLE_ERROR_PATTERN.test(errorMessage)) return false; if (isRetryableAssistantError(message)) return true; + // Same LiveAgent extension as withStreamRetry: a transient relay 5xx (#608) + // or a user-defined pattern is worth trying a different provider for — a + // fallback relay holds an independent origin/key and may not hit the same + // Cloudflare edge. Matches pi-ai's existing 524 → failover-eligible behavior. + if (isExtensionRetryableError(message, retryExtension)) return true; return FAILOVER_EXTRA_ELIGIBLE_ERROR_PATTERN.test(errorMessage); } @@ -225,6 +234,13 @@ export type ProviderFailoverStreamOptions = { */ onCommitted?: (candidateIndex: number) => void; now?: () => number; + /** + * Per-call override for the retry-error extension used by the eligibility + * classifier. Defaults to the process-wide extension + * ({@link setRetryErrorExtension}); kept consistent with withStreamRetry so a + * transient error retryable by same-provider retry is also failover-eligible. + */ + retryExtension?: RetryErrorExtension; }; type TerminalEvent = Extract; @@ -368,6 +384,7 @@ export function withProviderFailover( } else if (event.reason !== "aborted") { terminalEligible = isFailoverEligibleAssistantError( terminalMessage(event) as AssistantMessage, + options?.retryExtension, ); if (terminalEligible) { recordFailoverTargetResult(candidate.key, false, config, now()); diff --git a/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts b/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts index aef4299d6..30a417e1c 100644 --- a/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts +++ b/crates/agent-gui/src/lib/providers/runtime/streamRetry.ts @@ -5,6 +5,7 @@ import { createAssistantMessageEventStream, isRetryableAssistantError, } from "@earendil-works/pi-ai"; +import { RETRYABLE_PRESET_HTTP_STATUS_CODES } from "@liveagent/ui/lib/settings/types"; export type { RetryAttemptRecord } from "@liveagent/ui/lib/chat/retryAttempts"; @@ -14,6 +15,83 @@ export const DEFAULT_STREAM_RETRY_MAX_ATTEMPTS = 6; const STREAM_RETRY_BASE_DELAY_MS = 200; const STREAM_RETRY_BACKOFF_FACTOR = 2; +/** + * Extra retry classification layered on top of pi-ai's `isRetryableAssistantError`. + * Driven by the user's global retry-error settings (see `RetryErrorSettings`): + * - `statusCodes`: HTTP status codes (preset toggles) the user wants retried + * beyond pi-ai's hardcoded set (which already covers 429/500/502/503/504/524). + * - `patterns`: free-text substrings matched case-insensitively against the error + * message, for relay/gateway wording pi-ai doesn't recognize. + * + * The default module extension enables every preset code (Cloudflare 520-527), + * so relays self-heal out of the box (#608) even before the settings + * layer syncs the user's choices in. + */ +export type RetryErrorExtension = { + statusCodes?: number[]; + patterns?: string[]; +}; + +const DEFAULT_RETRY_ERROR_EXTENSION: RetryErrorExtension = { + statusCodes: [...RETRYABLE_PRESET_HTTP_STATUS_CODES], + patterns: [], +}; + +let currentRetryErrorExtension: RetryErrorExtension = DEFAULT_RETRY_ERROR_EXTENSION; + +/** + * Replaces the process-wide retry-error extension. Called by the settings layer + * whenever `retryErrorSettings` changes; the extension is a pure function of + * settings, so stale state is impossible once the effect re-runs. Tests can + * pass `null` to restore the default. + */ +export function setRetryErrorExtension(extension: RetryErrorExtension | null): void { + currentRetryErrorExtension = extension ?? DEFAULT_RETRY_ERROR_EXTENSION; +} + +export function getRetryErrorExtension(): RetryErrorExtension { + return currentRetryErrorExtension; +} + +function buildStatusCodePattern(codes: readonly number[]): RegExp | undefined { + if (codes.length === 0) return undefined; + // Word-boundary-ish: match the number not as a substring of a larger number + // (so "520" doesn't match "5200"). `\D|$` keeps it simple and sufficient for + // status codes embedded in error text like "HTTP 525" or "525 SSL handshake". + return new RegExp(`(?:^|\\D)(?:${codes.join("|")})(?:\\D|$)`); +} + +/** + * Whether a failed assistant message matches the LiveAgent retry extension + * (preset HTTP status codes + user-defined substrings), independently of + * pi-ai's `isRetryableAssistantError`. Does not re-check pi-ai's own patterns + * — callers OR the two together so the union is retryable. + */ +export function isExtensionRetryableError( + message: AssistantMessage | undefined, + extension: RetryErrorExtension = currentRetryErrorExtension, +): boolean { + if (!message) return false; + const errorMessage = (message as { errorMessage?: string }).errorMessage ?? ""; + if (!errorMessage) return false; + + const codes = extension.statusCodes; + if (codes && codes.length > 0) { + const pattern = buildStatusCodePattern(codes); + if (pattern?.test(errorMessage)) return true; + } + const patterns = extension.patterns; + if (patterns) { + const lower = errorMessage.toLowerCase(); + for (const raw of patterns) { + if (typeof raw !== "string") continue; + const needle = raw.trim(); + if (needle && lower.includes(needle.toLowerCase())) return true; + } + } + return false; +} + export type StreamRetryConfig = { maxAttempts?: number; disabled?: boolean; @@ -24,6 +102,12 @@ export type StreamRetryConfig = { onRetry?: (attempt: number, maxAttempts: number, errorMessage: string) => void; /** Invoked once a retried attempt commits its first content-bearing event. */ onRetryRecovered?: () => void; + /** + * Per-call override for the retry-error extension. Defaults to the + * process-wide extension set via `setRetryErrorExtension`; tests pass this + * to exercise the classifier without touching shared module state. + */ + retryExtension?: RetryErrorExtension; }; export type StreamRetryOptions = StreamRetryConfig & { @@ -144,7 +228,14 @@ export function withStreamRetry( } if (terminal?.type === "error" && !committed && !disabled && attempt < maxAttempts) { - if (isRetryableAssistantError(terminalMessage(terminal))) { + const failedMessage = terminalMessage(terminal); + // pi-ai's classifier first (preserves its non-retryable quota/billing + // guard), then LiveAgent's extension: preset HTTP status codes (Cloudflare + // 520-527 for relays, #608) + user-defined substrings from settings. + if ( + isRetryableAssistantError(failedMessage) || + isExtensionRetryableError(failedMessage, options?.retryExtension) + ) { const errorMessage = terminalMessage(terminal)?.errorMessage || "Unknown error"; attempt += 1; options?.onRetry?.(attempt - 1, maxAttempts - 1, errorMessage); diff --git a/crates/agent-gui/src/lib/settings/storage.ts b/crates/agent-gui/src/lib/settings/storage.ts index 9d9041822..d1ec72268 100644 --- a/crates/agent-gui/src/lib/settings/storage.ts +++ b/crates/agent-gui/src/lib/settings/storage.ts @@ -50,6 +50,7 @@ type LocalUiSettings = { updates?: unknown; selectedModel?: unknown; modelFailover?: unknown; + retryErrorSettings?: unknown; theme?: unknown; locale?: unknown; closeWindowBehavior?: unknown; @@ -98,6 +99,12 @@ function readLocalUiSettings(): { * with no providers would drop the whole queue. */ modelFailover: unknown; + /** + * Retry-error config is a local UI preference (not gateway-synced), so it + * lives in localStorage like chatRuntimeControls. Read raw; normalizeSettings + * validates preset codes and de-dupes custom patterns. + */ + retryErrorSettings: unknown; theme: Theme; locale: Locale; closeWindowBehavior: CloseWindowBehavior; @@ -137,6 +144,7 @@ function readLocalUiSettings(): { updates: defaults.updates, selectedModel: defaults.selectedModel, modelFailover: defaults.modelFailover, + retryErrorSettings: defaults.retryErrorSettings, theme: defaults.theme, locale: defaults.locale, closeWindowBehavior: defaults.closeWindowBehavior, @@ -157,6 +165,7 @@ function readLocalUiSettings(): { updates: normalizeUpdateSettings(parsed?.updates ?? defaults.updates), selectedModel: normalizeSelectedModel(parsed?.selectedModel), modelFailover: parsed?.modelFailover ?? defaults.modelFailover, + retryErrorSettings: parsed?.retryErrorSettings ?? defaults.retryErrorSettings, theme: normalizeTheme(parsed?.theme ?? defaults.theme), locale: normalizeLocale(hasStoredLocale ? parsed?.locale : defaults.locale), closeWindowBehavior: normalizeCloseWindowBehavior( @@ -171,6 +180,7 @@ function readLocalUiSettings(): { updates: defaults.updates, selectedModel: defaults.selectedModel, modelFailover: defaults.modelFailover, + retryErrorSettings: defaults.retryErrorSettings, theme: defaults.theme, locale: defaults.locale, closeWindowBehavior: defaults.closeWindowBehavior, @@ -189,6 +199,7 @@ function writeLocalUiSettings( | "theme" | "locale" | "closeWindowBehavior" + | "retryErrorSettings" >, ) { const payload = { @@ -200,6 +211,7 @@ function writeLocalUiSettings( theme: settings.theme, locale: settings.locale, closeWindowBehavior: settings.closeWindowBehavior, + retryErrorSettings: settings.retryErrorSettings, }; localStorage.setItem(LOCAL_UI_SETTINGS_STORAGE_KEY, JSON.stringify(payload)); } @@ -265,6 +277,7 @@ export async function loadPersistedSettingsWithDefaults(): Promise { assert.equal(DEFAULT_STREAM_RETRY_MAX_ATTEMPTS, 6); }); + +// ---- LiveAgent retry-error extension (#608: Cloudflare 5xx from relays) ---- + +test("isExtensionRetryableError matches a preset status code embedded in an error message", () => { + const message = createAssistant(undefined, "error", { errorMessage: "HTTP 525 SSL handshake failed" }); + assert.equal(isExtensionRetryableError(message, { statusCodes: [525] }), true); +}); + +test("isExtensionRetryableError does not match a status code that's a substring of a larger number", () => { + // "5200" must not match preset 520 (word-boundary guard). + const message = createAssistant(undefined, "error", { errorMessage: "upstream returned 5200" }); + assert.equal(isExtensionRetryableError(message, { statusCodes: [520] }), false); +}); + +test("isExtensionRetryableError matches a custom substring case-insensitively", () => { + const message = createAssistant(undefined, "error", { errorMessage: "Upstream SSL Handshake Failed" }); + assert.equal( + isExtensionRetryableError(message, { statusCodes: [], patterns: ["ssl handshake failed"] }), + true, + ); +}); + +test("isExtensionRetryableError returns false for an unrelated error and an empty extension", () => { + const message = createAssistant(undefined, "error", { errorMessage: "insufficient_quota: billing" }); + assert.equal(isExtensionRetryableError(message, { statusCodes: [], patterns: [] }), false); +}); + +test("withStreamRetry retries a Cloudflare 525 (#608) via an explicit retryExtension", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + if (calls < 2) return createErrorStream("HTTP 525 SSL Handshake Failed"); + return createSuccessStream("recovered"); + }, + { maxAttempts: 5, retryExtension: { statusCodes: [525] } }, + ); + await collectEvents(wrapped); + assert.equal(calls, 2); + const final = await wrapped.result(); + assert.equal(final.stopReason, "stop"); +}); + +test("withStreamRetry retries a user-defined custom substring pattern", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + if (calls < 2) return createErrorStream("origin SSL handshake with upstream failed"); + return createSuccessStream("ok"); + }, + { maxAttempts: 5, retryExtension: { statusCodes: [], patterns: ["SSL handshake"] } }, + ); + await collectEvents(wrapped); + assert.equal(calls, 2); +}); + +test("withStreamRetry does NOT retry a preset code the user has disabled", async () => { + // retryExtension omits 525 (user toggled it off) and carries nothing else that + // matches — pi-ai doesn't classify 525 either, so the turn must fail fast. + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return createErrorStream("HTTP 525 SSL Handshake Failed"); + }, + { maxAttempts: 5, retryExtension: { statusCodes: [], patterns: [] } }, + ); + await collectEvents(wrapped); + assert.equal(calls, 1); + const final = await wrapped.result(); + assert.equal(final.stopReason, "error"); +}); + +test("withStreamRetry still does not retry a non-retryable quota error when the extension doesn't match it", async () => { + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + return createErrorStream("insufficient_quota: billing required"); + }, + { maxAttempts: 5, retryExtension: { statusCodes: [525], patterns: ["SSL handshake"] } }, + ); + await collectEvents(wrapped); + assert.equal(calls, 1); +}); + +test("withStreamRetry retries a Cloudflare 525 via the module default extension (zero-config #608 fix)", async () => { + // Reset to the process-wide default (all presets on) so this test doesn't + // depend on ordering or prior tests mutating the module extension. + setRetryErrorExtension(null); + let calls = 0; + const wrapped = withStreamRetry( + () => { + calls += 1; + if (calls < 2) return createErrorStream("HTTP 525 SSL Handshake Failed"); + return createSuccessStream("recovered"); + }, + { maxAttempts: 5 }, + ); + await collectEvents(wrapped); + assert.equal(calls, 2); +}); + +test("isFailoverEligibleAssistantError treats a Cloudflare 525 as failover-eligible via the extension", () => { + const message = createAssistant(undefined, "error", { errorMessage: "HTTP 525 SSL Handshake Failed" }); + assert.equal(isFailoverEligibleAssistantError(message, { statusCodes: [525] }), true); +}); + +test("isFailoverEligibleAssistantError still rejects client-class errors even if the extension matches", () => { + // Context-overflow is ineligible regardless of extension — switching providers + // can't fix a too-long prompt. + const message = createAssistant(undefined, "error", { + errorMessage: "prompt is too long (520 context length exceeded)", + }); + assert.equal(isFailoverEligibleAssistantError(message, { statusCodes: [520] }), false); +}); + +// ---- Extension state + classifier edge cases ---- + +test("setRetryErrorExtension / getRetryErrorExtension round-trip and restore-to-default", () => { + // The default enables every Cloudflare preset, so a 525 matches it without + // any per-call extension. + const message = createAssistant(undefined, "error", { errorMessage: "HTTP 525" }); + assert.equal(isExtensionRetryableError(message), true); + + // A user who disabled every preset and added no patterns sees no match. + setRetryErrorExtension({ statusCodes: [], patterns: [] }); + assert.equal(isExtensionRetryableError(message), false); + assert.deepEqual(getRetryErrorExtension(), { statusCodes: [], patterns: [] }); + + // Custom patterns flow through the module extension too. + setRetryErrorExtension({ statusCodes: [], patterns: ["ssl handshake failed"] }); + assert.equal( + isExtensionRetryableError( + createAssistant(undefined, "error", { errorMessage: "upstream SSL Handshake Failed" }), + ), + true, + ); + + // null restores the default (all presets on) — the zero-config #608 fix. + setRetryErrorExtension(null); + assert.equal(isExtensionRetryableError(message), true); +}); + +test("isExtensionRetryableError returns false for undefined or empty error messages", () => { + assert.equal(isExtensionRetryableError(undefined, { statusCodes: [525] }), false); + assert.equal( + isExtensionRetryableError(createAssistant(undefined, "error"), { statusCodes: [525] }), + false, + ); + assert.equal( + isExtensionRetryableError( + createAssistant(undefined, "error", { errorMessage: "" }), + { statusCodes: [525] }, + ), + false, + ); +}); + +test("isExtensionRetryableError matches any one of several preset codes (alternation)", () => { + const ext = { statusCodes: [520, 521, 525], patterns: [] }; + for (const code of [520, 521, 525]) { + assert.equal( + isExtensionRetryableError( + createAssistant(undefined, "error", { errorMessage: `error ${code} occurred` }), + ext, + ), + true, + ); + } + // An unrelated 5xx not in the list is not matched by the code branch. + assert.equal( + isExtensionRetryableError( + createAssistant(undefined, "error", { errorMessage: "error 530 occurred" }), + ext, + ), + false, + ); +}); + +test("isExtensionRetryableError ignores whitespace-only custom patterns", () => { + const ext = { statusCodes: [], patterns: [" ", ""] }; + assert.equal( + isExtensionRetryableError( + createAssistant(undefined, "error", { errorMessage: "anything" }), + ext, + ), + false, + ); +}); diff --git a/crates/agent-gui/test/settings/retry-error-normalize.test.mjs b/crates/agent-gui/test/settings/retry-error-normalize.test.mjs new file mode 100644 index 000000000..0eb9aa9d5 --- /dev/null +++ b/crates/agent-gui/test/settings/retry-error-normalize.test.mjs @@ -0,0 +1,57 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { createTsModuleLoader } from "../helpers/load-ts-module.mjs"; + +const loader = createTsModuleLoader(); +const settings = loader.loadModule("src/lib/settings/index.ts"); + +test("retryErrorSettings defaults to every Cloudflare 5xx preset enabled, no custom patterns", () => { + const normalized = settings.normalizeRetryErrorSettings({}); + assert.deepEqual([...normalized.presetStatusCodes].sort(), [520, 521, 522, 523, 525, 526, 527]); + assert.deepEqual(normalized.customPatterns, []); +}); + +test("presetStatusCodes are validated against the known preset list and deduped", () => { + const normalized = settings.normalizeRetryErrorSettings({ + presetStatusCodes: [525, 525, 520, 999, "521", null], + }); + // 999 is unknown → dropped; "521"/null are non-numbers → dropped; 525 deduped. + assert.deepEqual(normalized.presetStatusCodes.sort((a, b) => a - b), [520, 525]); +}); + +test("a present-but-empty presetStatusCodes is respected (user opts out of all presets)", () => { + const normalized = settings.normalizeRetryErrorSettings({ presetStatusCodes: [] }); + assert.deepEqual(normalized.presetStatusCodes, []); +}); + +test("a missing presetStatusCodes field falls back to all presets (legacy snapshot)", () => { + const normalized = settings.normalizeRetryErrorSettings({ customPatterns: ["x"] }); + assert.deepEqual([...normalized.presetStatusCodes].sort(), [520, 521, 522, 523, 525, 526, 527]); +}); + +test("customPatterns are trimmed, de-duped case-insensitively, and empties dropped", () => { + const normalized = settings.normalizeRetryErrorSettings({ + customPatterns: ["SSL handshake failed", " ssl handshake failed ", "", " ", "525"], + }); + assert.deepEqual(normalized.customPatterns, ["SSL handshake failed", "525"]); +}); + +test("normalizeSettings carries retryErrorSettings and normalizes a raw snapshot", () => { + const normalized = settings.normalizeSettings({ + retryErrorSettings: { + presetStatusCodes: [525, 999, 525], + customPatterns: ["x", "x"], + }, + }); + assert.deepEqual(normalized.retryErrorSettings.presetStatusCodes, [525]); + assert.deepEqual(normalized.retryErrorSettings.customPatterns, ["x"]); +}); + +test("normalizeSettings fills retryErrorSettings defaults when the field is absent (legacy)", () => { + const normalized = settings.normalizeSettings({}); + assert.deepEqual( + [...normalized.retryErrorSettings.presetStatusCodes].sort(), + [520, 521, 522, 523, 525, 526, 527], + ); + assert.deepEqual(normalized.retryErrorSettings.customPatterns, []); +}); diff --git a/crates/agent-gui/test/settings/storage.test.mjs b/crates/agent-gui/test/settings/storage.test.mjs index afbd052c0..f8197ebfe 100644 --- a/crates/agent-gui/test/settings/storage.test.mjs +++ b/crates/agent-gui/test/settings/storage.test.mjs @@ -54,3 +54,77 @@ test("legacy local storage treats a null locale as an invalid saved preference", }); }); }); + +// retryErrorSettings is a local-only UI preference persisted in localStorage +// (not gateway-synced), so its read/write round-trip lives entirely in the +// local-ui settings path — no backend command is involved. + +test("retryErrorSettings default to every Cloudflare preset when localStorage is empty", async () => { + const localStorage = createMemoryLocalStorage(); + + await withGlobal("navigator", { languages: ["en-US"], language: "en-US" }, async () => { + await withGlobal("localStorage", localStorage, async () => { + const loader = createTsModuleLoader({ + mocks: { "@tauri-apps/api/core": { invoke: async () => ({}) } }, + }); + const storage = loader.loadModule("src/lib/settings/storage.ts"); + + const loaded = await storage.loadPersistedSettings(); + assert.deepEqual( + [...loaded.retryErrorSettings.presetStatusCodes].sort((a, b) => a - b), + [520, 521, 522, 523, 525, 526, 527], + ); + assert.deepEqual(loaded.retryErrorSettings.customPatterns, []); + }); + }); +}); + +test("retryErrorSettings are read back from localStorage and normalized", async () => { + const localStorage = createMemoryLocalStorage({ + [LOCAL_UI_SETTINGS_STORAGE_KEY]: JSON.stringify({ + retryErrorSettings: { + // User disabled 520/521, kept 525; an unknown code (999) and a + // duplicate must be dropped on read. + presetStatusCodes: [525, 525, 999], + customPatterns: ["SSL handshake failed", " ssl handshake failed ", ""], + }, + }), + }); + + await withGlobal("navigator", { languages: ["en-US"], language: "en-US" }, async () => { + await withGlobal("localStorage", localStorage, async () => { + const loader = createTsModuleLoader({ + mocks: { "@tauri-apps/api/core": { invoke: async () => ({}) } }, + }); + const storage = loader.loadModule("src/lib/settings/storage.ts"); + + const loaded = await storage.loadPersistedSettings(); + assert.deepEqual(loaded.retryErrorSettings.presetStatusCodes, [525]); + // Case-insensitive de-dup + trim + empty-drop. + assert.deepEqual(loaded.retryErrorSettings.customPatterns, ["SSL handshake failed"]); + }); + }); +}); + +test("a missing retryErrorSettings field falls back to all presets (legacy snapshot)", async () => { + // A pre-feature localStorage blob has no retryErrorSettings key; it must + // normalize to the all-presets-on default, not an empty config. + const localStorage = createMemoryLocalStorage({ + [LOCAL_UI_SETTINGS_STORAGE_KEY]: JSON.stringify({ theme: "dark" }), + }); + + await withGlobal("navigator", { languages: ["en-US"], language: "en-US" }, async () => { + await withGlobal("localStorage", localStorage, async () => { + const loader = createTsModuleLoader({ + mocks: { "@tauri-apps/api/core": { invoke: async () => ({}) } }, + }); + const storage = loader.loadModule("src/lib/settings/storage.ts"); + + const loaded = await storage.loadPersistedSettings(); + assert.deepEqual( + [...loaded.retryErrorSettings.presetStatusCodes].sort((a, b) => a - b), + [520, 521, 522, 523, 525, 526, 527], + ); + }); + }); +}); diff --git a/crates/agent-ui/src/i18n/translations/enUSSettings.ts b/crates/agent-ui/src/i18n/translations/enUSSettings.ts index b0c77270a..6a3f4588e 100644 --- a/crates/agent-ui/src/i18n/translations/enUSSettings.ts +++ b/crates/agent-ui/src/i18n/translations/enUSSettings.ts @@ -661,6 +661,26 @@ export const EN_US_SETTINGS_TRANSLATIONS = { "settings.failoverCooldownSeconds": "Cooldown (seconds)", "settings.failoverCooldownSecondsHint": "How long an open circuit skips the provider before allowing a probe request (5-3600 seconds).", + "settings.retryError": "Retry Error Config", + "settings.retryErrorDesc": + "Choose which upstream errors the stream-retry loop should treat as transient and retry with exponential backoff. Useful for relay/proxy stations that return intermittent Cloudflare 5xx.", + "settings.retryErrorPresets": "Preset status codes", + "settings.retryErrorBuiltinNote": + "429 / 500 / 502 / 503 / 504 / 524 and common network/timeout errors are already retried by the built-in engine, regardless of these toggles.", + "settings.retryError.preset.520": "Web Server Returned an Unknown Error (Cloudflare 520)", + "settings.retryError.preset.521": "Web Server Is Down (Cloudflare 521)", + "settings.retryError.preset.522": "Connection Timed Out (Cloudflare 522)", + "settings.retryError.preset.523": "Origin Is Unreachable (Cloudflare 523)", + "settings.retryError.preset.525": "SSL Handshake Failed (Cloudflare 525)", + "settings.retryError.preset.526": "Invalid SSL Certificate (Cloudflare 526)", + "settings.retryError.preset.527": "Railgun Request Error (Cloudflare 527)", + "settings.retryErrorCustomPatterns": "Custom error keywords", + "settings.retryErrorCustomPatternsDesc": + "Add substrings matched case-insensitively against the error message. Any error containing one of these is retried. e.g. \"SSL handshake failed\".", + "settings.retryErrorCustomPatternPlaceholder": "e.g. SSL handshake failed", + "settings.retryErrorAddPattern": "Add", + "settings.retryErrorRemovePattern": "Remove", + "settings.retryErrorCustomPatternEmpty": "No custom keywords. Add one to extend retry to relay/gateway wording the built-in engine doesn't recognize.", "settings.agentsTitle": "Prompt Templates", "settings.agentsDesc": "Manage global templates and project-specific prompts in one place", "settings.agentsGlobalTab": "Global templates", diff --git a/crates/agent-ui/src/i18n/translations/zhCNSettings.ts b/crates/agent-ui/src/i18n/translations/zhCNSettings.ts index b194612cc..a12c8cb50 100644 --- a/crates/agent-ui/src/i18n/translations/zhCNSettings.ts +++ b/crates/agent-ui/src/i18n/translations/zhCNSettings.ts @@ -635,6 +635,26 @@ export const ZH_CN_SETTINGS_TRANSLATIONS = { "settings.failoverCooldownSeconds": "熔断冷却时间（秒）", "settings.failoverCooldownSecondsHint": "熔断后跳过该供应商的时长，到期后放行探测请求（5-3600 秒）。", + "settings.retryError": "重试错误配置", + "settings.retryErrorDesc": + "选择哪些上游报错应被流式重试回路视为瞬时错误并按指数退避自动重试。适用于会偶发 Cloudflare 5xx 的中转站/代理。", + "settings.retryErrorPresets": "预设状态码", + "settings.retryErrorBuiltinNote": + "429 / 500 / 502 / 503 / 504 / 524 以及常见的网络/超时错误已由内置引擎自动重试,不受这些开关影响。", + "settings.retryError.preset.520": "Web 服务器返回未知错误 (Cloudflare 520)", + "settings.retryError.preset.521": "Web 服务器已宕机 (Cloudflare 521)", + "settings.retryError.preset.522": "连接超时 (Cloudflare 522)", + "settings.retryError.preset.523": "源站不可达 (Cloudflare 523)", + "settings.retryError.preset.525": "SSL 握手失败 (Cloudflare 525)", + "settings.retryError.preset.526": "SSL 证书无效 (Cloudflare 526)", + "settings.retryError.preset.527": "Railgun 请求错误 (Cloudflare 527)", + "settings.retryErrorCustomPatterns": "自定义错误关键词", + "settings.retryErrorCustomPatternsDesc": + "添加大小写不敏感的子串,与错误信息匹配。包含任一关键词的报错都会被重试,例如 \"SSL handshake failed\"。", + "settings.retryErrorCustomPatternPlaceholder": "例如:SSL handshake failed", + "settings.retryErrorAddPattern": "添加", + "settings.retryErrorRemovePattern": "移除", + "settings.retryErrorCustomPatternEmpty": "暂无自定义关键词。添加后可让内置引擎不识别的中转站/网关措辞也纳入重试。", "settings.agentsTitle": "提示词模板", "settings.agentsDesc": "统一管理全局模板与每个项目的专属提示词", "settings.agentsGlobalTab": "全局模板", diff --git a/crates/agent-ui/src/lib/settings/index.ts b/crates/agent-ui/src/lib/settings/index.ts index 9c7dc6056..9fe979572 100644 --- a/crates/agent-ui/src/lib/settings/index.ts +++ b/crates/agent-ui/src/lib/settings/index.ts @@ -35,6 +35,7 @@ import { MIN_CHAT_TRANSCRIPT_WIDTH, } from "@liveagent/ui/lib/transcript-width/transcriptWidthModel"; import { normalizeModelFailoverSettings } from "./modelFailover"; +import { normalizeRetryErrorSettings } from "./retryError"; import { normalizeChatTranscriptSettings, normalizeFontScaleSettings, @@ -141,6 +142,7 @@ export { normalizeModelFailoverSettings, normalizeProviderFailoverSettings, } from "./modelFailover"; +export { normalizeRetryErrorSettings } from "./retryError"; export { normalizeChatTranscriptSettings, normalizeFontScale, @@ -1583,6 +1585,7 @@ export function getDefaultSettings(): AppSettings { memory: normalizeMemorySettings({}, customProviders), customSettings: normalizeCustomSettings({}, customProviders), modelFailover: normalizeModelFailoverSettings({}, customProviders), + retryErrorSettings: normalizeRetryErrorSettings({}), updates: normalizeUpdateSettings({}), skills: { enabled: true, @@ -1625,6 +1628,9 @@ export function normalizeSettings(input?: Partial | null): AppSetti obj.modelFailover ?? defaults.modelFailover, customProviders, ), + retryErrorSettings: normalizeRetryErrorSettings( + obj.retryErrorSettings ?? defaults.retryErrorSettings, + ), updates: normalizeUpdateSettings(obj.updates ?? defaults.updates), skills: normalizeSkillsSettings(obj.skills ?? defaults.skills), chatRuntimeControls: normalizeChatRuntimeControls( diff --git a/crates/agent-ui/src/lib/settings/retryError.ts b/crates/agent-ui/src/lib/settings/retryError.ts new file mode 100644 index 000000000..2a72fa3f9 --- /dev/null +++ b/crates/agent-ui/src/lib/settings/retryError.ts @@ -0,0 +1,57 @@ +import { + DEFAULT_RETRY_ERROR_SETTINGS, + type RetryErrorSettings, + RETRYABLE_PRESET_HTTP_STATUS_CODES, +} from "./types"; + +const PRESET_CODE_SET = new Set(RETRYABLE_PRESET_HTTP_STATUS_CODES); + +/** + * Normalizes the user-defined retry-error config. + * + * `presetStatusCodes` are validated against the known preset list (unknown codes + * dropped, duplicates removed). A present-but-empty array is respected — the + * user may want to opt out of every preset — while a missing field (legacy + * snapshot) falls back to all presets on, so relays self-heal out of the box + * (#608) without requiring the user to opt in. + * + * `customPatterns` are trimmed, de-duplicated case-insensitively, and empties + * are dropped. + */ +export function normalizeRetryErrorSettings(input: unknown): RetryErrorSettings { + const obj = (input && typeof input === "object" ? input : {}) as Record; + const defaults = DEFAULT_RETRY_ERROR_SETTINGS; + + const presetStatusCodes: number[] = []; + if (Array.isArray(obj.presetStatusCodes)) { + const seen = new Set(); + for (const raw of obj.presetStatusCodes) { + const code = + typeof raw === "number" && Number.isFinite(raw) ? Math.round(raw) : Number.NaN; + if (!Number.isFinite(code) || !PRESET_CODE_SET.has(code) || seen.has(code)) continue; + seen.add(code); + presetStatusCodes.push(code); + } + } else { + // Missing field (legacy snapshot): default to every preset enabled. + presetStatusCodes.push(...defaults.presetStatusCodes); + } + + const customPatterns: string[] = []; + if (Array.isArray(obj.customPatterns)) { + const seen = new Set(); + for (const raw of obj.customPatterns) { + if (typeof raw !== "string") continue; + const trimmed = raw.trim(); + const key = trimmed.toLowerCase(); + if (!trimmed || seen.has(key)) continue; + seen.add(key); + customPatterns.push(trimmed); + } + } + + return { + presetStatusCodes, + customPatterns, + }; +} diff --git a/crates/agent-ui/src/lib/settings/types.ts b/crates/agent-ui/src/lib/settings/types.ts index 882f24f98..191776a84 100644 --- a/crates/agent-ui/src/lib/settings/types.ts +++ b/crates/agent-ui/src/lib/settings/types.ts @@ -215,6 +215,41 @@ export function getDefaultModelFailoverSettings(): ModelFailoverSettings { }; } +/** + * Cloudflare 5xx status codes that relays surface when their origin + * errors. pi-ai's `isRetryableAssistantError` already retries 524; these are + * the rest of Cloudflare's transient 5xx family (#608). Offered as toggleable + * presets in the settings UI; the runtime retries any error message that + * contains the code as a standalone number. + */ +export const RETRYABLE_PRESET_HTTP_STATUS_CODES = [ + 520, 521, 522, 523, 525, 526, 527, +] as const; + +/** + * User-defined retry-error classification, layered on top of pi-ai's + * `isRetryableAssistantError`. Lets users decide which errors the stream-retry + * loop should treat as transient (#608) — preset Cloudflare 5xx toggles plus + * free-text substrings for relay/gateway wording pi-ai doesn't recognize. + */ +export type RetryErrorSettings = { + /** + * HTTP status codes (from `RETRYABLE_PRESET_HTTP_STATUS_CODES`) the user has + * enabled. Defaults to all presets on so relays self-heal out of the box. + */ + presetStatusCodes: number[]; + /** + * Free-text substrings matched case-insensitively against the error message. + * An error containing any of these is retried. e.g. "SSL handshake failed". + */ + customPatterns: string[]; +}; + +export const DEFAULT_RETRY_ERROR_SETTINGS: RetryErrorSettings = { + presetStatusCodes: [...RETRYABLE_PRESET_HTTP_STATUS_CODES], + customPatterns: [], +}; + export type SystemProxyType = "socks5" | "http"; // 系统级出站代理：注入本地 shell 命令 env，并供勾选了 useSystemProxy 的 @@ -574,6 +609,7 @@ export type AppSettings = { memory: MemorySettings; customSettings: CustomSettings; modelFailover: ModelFailoverSettings; + retryErrorSettings: RetryErrorSettings; updates: UpdateSettings; skills: SkillsSettings; chatRuntimeControls: ChatRuntimeControls; diff --git a/crates/agent-ui/src/pages/settings/ProvidersSection.tsx b/crates/agent-ui/src/pages/settings/ProvidersSection.tsx index 8807d70f1..1467c6fdf 100644 --- a/crates/agent-ui/src/pages/settings/ProvidersSection.tsx +++ b/crates/agent-ui/src/pages/settings/ProvidersSection.tsx @@ -51,6 +51,7 @@ import { UsagePlanLine, usageRelativeTimeText, } from "./ProviderPresentation"; +import { RetryErrorSection } from "./RetryErrorSection"; function FailoverNumberField(props: { label: string; @@ -451,6 +452,7 @@ function CustomSettingsDrawer( setSettings={setSettings} providerType={providerType} /> + diff --git a/crates/agent-ui/src/pages/settings/RetryErrorSection.tsx b/crates/agent-ui/src/pages/settings/RetryErrorSection.tsx new file mode 100644 index 000000000..01a89110e --- /dev/null +++ b/crates/agent-ui/src/pages/settings/RetryErrorSection.tsx @@ -0,0 +1,174 @@ +// Retry-error config: let the user define which upstream errors the stream-retry +// loop should treat as retryable. +// +// Background: relay/proxy stations (Cloudflare-fronted) intermittently return +// 520/521/525 and similar transient 5xx. Before #608 these were not retried, so +// the request failed outright. pi-ai's isRetryableAssistantError covers the +// common codes 429/500/502/503/504/524, but not the Cloudflare 5xx relays emit. +// +// This section exposes the "retry-error extension" LiveAgent layers on top of +// pi-ai for the user to configure: +// 1. Preset status code toggles (Cloudflare 520-527) — all on by default, so +// #608 is fixed out of the box; +// 2. Custom error keywords (case-insensitive substrings) — covers relay/gateway +// wording pi-ai doesn't recognize, e.g. "SSL handshake failed". +// The runtime layers both onto streamRetry's and providerFailover's retryable +// classification. Local UI preference only (localStorage), not gateway-synced. + +import { Button } from "@liveagent/ui/components/ui/button"; +import { Input } from "@liveagent/ui/components/ui/input"; +import { Label } from "@liveagent/ui/components/ui/label"; +import { Switch } from "@liveagent/ui/components/ui/switch"; +import { useLocale } from "@liveagent/ui/i18n/index"; +import { useState } from "react"; +import type { SettingsSectionProps } from "@liveagent/app/pages/settings/types"; +import { RETRYABLE_PRESET_HTTP_STATUS_CODES } from "@liveagent/ui/lib/settings/types"; + +export function RetryErrorSection(props: SettingsSectionProps) { + const { settings, setSettings } = props; + const { t } = useLocale(); + const retryErrorSettings = settings.retryErrorSettings; + const [patternDraft, setPatternDraft] = useState(""); + + function isPresetEnabled(code: number): boolean { + return retryErrorSettings.presetStatusCodes.includes(code); + } + + function togglePresetCode(code: number, enabled: boolean) { + setSettings((prev) => { + const current = prev.retryErrorSettings.presetStatusCodes; + const next = enabled + ? current.includes(code) + ? current + : [...current, code] + : current.filter((item) => item !== code); + return { + ...prev, + retryErrorSettings: { + ...prev.retryErrorSettings, + presetStatusCodes: next, + }, + }; + }); + } + + function addPattern() { + const trimmed = patternDraft.trim(); + if (!trimmed) return; + setPatternDraft(""); + setSettings((prev) => ({ + ...prev, + retryErrorSettings: { + ...prev.retryErrorSettings, + // normalizeSettings de-dupes case-insensitively and drops empties. + customPatterns: [...prev.retryErrorSettings.customPatterns, trimmed], + }, + })); + } + + function removePattern(pattern: string) { + setSettings((prev) => ({ + ...prev, + retryErrorSettings: { + ...prev.retryErrorSettings, + customPatterns: prev.retryErrorSettings.customPatterns.filter( + (item) => item !== pattern, + ), + }, + })); + } + + return ( +
+
+
+ +

+ {t("settings.retryErrorDesc")} +

+
+
+ + {/* Preset Cloudflare 5xx toggles */} +
+ +
+ {RETRYABLE_PRESET_HTTP_STATUS_CODES.map((code) => ( +
+
+
+ + {code} + + + {t(`settings.retryError.preset.${code}`)} + +
+
+ togglePresetCode(code, checked === true)} + /> +
+ ))} +
+

+ {t("settings.retryErrorBuiltinNote")} +

+
+ + {/* Custom error patterns */} +
+ +

+ {t("settings.retryErrorCustomPatternsDesc")} +

+
+ setPatternDraft(event.target.value)} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + addPattern(); + } + }} + /> + +
+ {retryErrorSettings.customPatterns.length > 0 ? ( +
+ {retryErrorSettings.customPatterns.map((pattern) => ( + + ))} +
+ ) : ( +

+ {t("settings.retryErrorCustomPatternEmpty")} +

+ )} +
+
+ ); +}