feat(retry): auto-retry Cloudflare 5xx from relays + user-configurable retry-error settings

crates/agent-gui/src/App.tsx

@@ -27,6 +27,7 @@ import { AppBootShell } from "./components/app/AppBootShell";
 import { useNativeInputContextMenu } from "./components/input-context-menu/NativeInputContextMenu";
 import { WindowsTitleBar } from "./components/WindowsTitleBar";
 import { useAppUpdateController } from "./lib/appUpdates";
+import { setRetryErrorExtension } from "./lib/providers/runtime/streamRetry";
 import {
   type AppSettings,
   getDefaultSettings,
@@ -387,6 +388,18 @@ export default function App() {
     return () => window.clearTimeout(timeoutId);
   }, [settingsReady]);
 
+  // Push the user's retry-error classification (preset Cloudflare 5xx toggles +
+  // custom substrings) into the stream-retry runtime. The extension is a pure
+  // function of settings, so re-running on every change keeps the runtime in
+  // sync without any per-call plumbing. The runtime's default already enables
+  // every preset, so this is a no-op until the user actually changes something.
+  useEffect(() => {
+    setRetryErrorExtension({
+      statusCodes: settings.retryErrorSettings.presetStatusCodes,
+      patterns: settings.retryErrorSettings.customPatterns,
+    });
+  }, [settings.retryErrorSettings]);
+
   const queueSettingsSave = useCallback(
     (prev: AppSettings, next: AppSettings, fallback: string, publishSync: boolean) => {
       const saveSequence = ++saveSequenceRef.current;

crates/agent-gui/src/lib/providers/runtime/providerFailover.ts

@@ -5,6 +5,7 @@ import {
   createAssistantMessageEventStream,
   isRetryableAssistantError,
 } from "@earendil-works/pi-ai";
+import { isExtensionRetryableError, type RetryErrorExtension } from "./streamRetry";
 
 /**
  * Provider auto-failover runtime (cc-switch inspired).
@@ -182,11 +183,19 @@ const FAILOVER_EXTRA_ELIGIBLE_ERROR_PATTERN = new RegExp(
  * style client errors never fail over even though they may contain digits
  * that look like status codes.
  */
-export function isFailoverEligibleAssistantError(message: AssistantMessage | undefined): boolean {
+export function isFailoverEligibleAssistantError(
+  message: AssistantMessage | undefined,
+  retryExtension?: RetryErrorExtension,
+): boolean {
   if (!message) return false;
   const errorMessage = (message as { errorMessage?: string }).errorMessage ?? "";
   if (FAILOVER_INELIGIBLE_ERROR_PATTERN.test(errorMessage)) return false;
   if (isRetryableAssistantError(message)) return true;
+  // Same LiveAgent extension as withStreamRetry: a transient relay 5xx (#608)
+  // or a user-defined pattern is worth trying a different provider for — a
+  // fallback relay holds an independent origin/key and may not hit the same
+  // Cloudflare edge. Matches pi-ai's existing 524 → failover-eligible behavior.
+  if (isExtensionRetryableError(message, retryExtension)) return true;
   return FAILOVER_EXTRA_ELIGIBLE_ERROR_PATTERN.test(errorMessage);
 }
 
@@ -225,6 +234,13 @@ export type ProviderFailoverStreamOptions = {
    */
   onCommitted?: (candidateIndex: number) => void;
   now?: () => number;
+  /**
+   * Per-call override for the retry-error extension used by the eligibility
+   * classifier. Defaults to the process-wide extension
+   * ({@link setRetryErrorExtension}); kept consistent with withStreamRetry so a
+   * transient error retryable by same-provider retry is also failover-eligible.
+   */
+  retryExtension?: RetryErrorExtension;
 };
 
 type TerminalEvent = Extract<AssistantMessageEvent, { type: "done" | "error" }>;
@@ -368,6 +384,7 @@ export function withProviderFailover(
           } else if (event.reason !== "aborted") {
             terminalEligible = isFailoverEligibleAssistantError(
               terminalMessage(event) as AssistantMessage,
+              options?.retryExtension,
             );
             if (terminalEligible) {
               recordFailoverTargetResult(candidate.key, false, config, now());

crates/agent-gui/src/lib/providers/runtime/streamRetry.ts

@@ -5,6 +5,7 @@ import {
   createAssistantMessageEventStream,
   isRetryableAssistantError,
 } from "@earendil-works/pi-ai";
+import { RETRYABLE_PRESET_HTTP_STATUS_CODES } from "@liveagent/ui/lib/settings/types";
 
 export type { RetryAttemptRecord } from "@liveagent/ui/lib/chat/retryAttempts";
 
@@ -14,6 +15,83 @@ export const DEFAULT_STREAM_RETRY_MAX_ATTEMPTS = 6;
 const STREAM_RETRY_BASE_DELAY_MS = 200;
 const STREAM_RETRY_BACKOFF_FACTOR = 2;
 
+/**
+ * Extra retry classification layered on top of pi-ai's `isRetryableAssistantError`.
+ * Driven by the user's global retry-error settings (see `RetryErrorSettings`):
+ * - `statusCodes`: HTTP status codes (preset toggles) the user wants retried
+ *   beyond pi-ai's hardcoded set (which already covers 429/500/502/503/504/524).
+ * - `patterns`: free-text substrings matched case-insensitively against the error
+ *   message, for relay/gateway wording pi-ai doesn't recognize.
+ *
+ * The default module extension enables every preset code (Cloudflare 520-527),
+ * so relays self-heal out of the box (#608) even before the settings
+ * layer syncs the user's choices in.
+ */
+export type RetryErrorExtension = {
+  statusCodes?: number[];
+  patterns?: string[];
+};
+
+const DEFAULT_RETRY_ERROR_EXTENSION: RetryErrorExtension = {
+  statusCodes: [...RETRYABLE_PRESET_HTTP_STATUS_CODES],
+  patterns: [],
+};
+
+let currentRetryErrorExtension: RetryErrorExtension = DEFAULT_RETRY_ERROR_EXTENSION;
+
+/**
+ * Replaces the process-wide retry-error extension. Called by the settings layer
+ * whenever `retryErrorSettings` changes; the extension is a pure function of
+ * settings, so stale state is impossible once the effect re-runs. Tests can
+ * pass `null` to restore the default.
+ */
+export function setRetryErrorExtension(extension: RetryErrorExtension | null): void {
+  currentRetryErrorExtension = extension ?? DEFAULT_RETRY_ERROR_EXTENSION;
+}
+
+export function getRetryErrorExtension(): RetryErrorExtension {
+  return currentRetryErrorExtension;
+}
+
+function buildStatusCodePattern(codes: readonly number[]): RegExp | undefined {
+  if (codes.length === 0) return undefined;
+  // Word-boundary-ish: match the number not as a substring of a larger number
+  // (so "520" doesn't match "5200"). `\D|$` keeps it simple and sufficient for
+  // status codes embedded in error text like "HTTP 525" or "525 SSL handshake".
+  return new RegExp(`(?:^|\\D)(?:${codes.join("|")})(?:\\D|$)`);
+}
+
+/**
+ * Whether a failed assistant message matches the LiveAgent retry extension
+ * (preset HTTP status codes + user-defined substrings), independently of
+ * pi-ai's `isRetryableAssistantError`. Does not re-check pi-ai's own patterns
+ * — callers OR the two together so the union is retryable.
+ */
+export function isExtensionRetryableError(
+  message: AssistantMessage | undefined,
+  extension: RetryErrorExtension = currentRetryErrorExtension,
+): boolean {
+  if (!message) return false;
+  const errorMessage = (message as { errorMessage?: string }).errorMessage ?? "";
+  if (!errorMessage) return false;
+
+  const codes = extension.statusCodes;
+  if (codes && codes.length > 0) {
+    const pattern = buildStatusCodePattern(codes);
+    if (pattern?.test(errorMessage)) return true;
+  }
+  const patterns = extension.patterns;
+  if (patterns) {
+    const lower = errorMessage.toLowerCase();
+    for (const raw of patterns) {
+      if (typeof raw !== "string") continue;
+      const needle = raw.trim();
+      if (needle && lower.includes(needle.toLowerCase())) return true;
+    }
+  }
+  return false;
+}
+
 export type StreamRetryConfig = {
   maxAttempts?: number;
   disabled?: boolean;
@@ -24,6 +102,12 @@ export type StreamRetryConfig = {
   onRetry?: (attempt: number, maxAttempts: number, errorMessage: string) => void;
   /** Invoked once a retried attempt commits its first content-bearing event. */
   onRetryRecovered?: () => void;
+  /**
+   * Per-call override for the retry-error extension. Defaults to the
+   * process-wide extension set via `setRetryErrorExtension`; tests pass this
+   * to exercise the classifier without touching shared module state.
+   */
+  retryExtension?: RetryErrorExtension;
 };
 
 export type StreamRetryOptions = StreamRetryConfig & {
@@ -144,7 +228,14 @@ export function withStreamRetry(
       }
 
       if (terminal?.type === "error" && !committed && !disabled && attempt < maxAttempts) {
-        if (isRetryableAssistantError(terminalMessage(terminal))) {
+        const failedMessage = terminalMessage(terminal);
+        // pi-ai's classifier first (preserves its non-retryable quota/billing
+        // guard), then LiveAgent's extension: preset HTTP status codes (Cloudflare
+        // 520-527 for relays, #608) + user-defined substrings from settings.
+        if (
+          isRetryableAssistantError(failedMessage) ||
+          isExtensionRetryableError(failedMessage, options?.retryExtension)
+        ) {
           const errorMessage = terminalMessage(terminal)?.errorMessage || "Unknown error";
           attempt += 1;
           options?.onRetry?.(attempt - 1, maxAttempts - 1, errorMessage);

crates/agent-gui/src/lib/settings/storage.ts

@@ -50,6 +50,7 @@ type LocalUiSettings = {
   updates?: unknown;
   selectedModel?: unknown;
   modelFailover?: unknown;
+  retryErrorSettings?: unknown;
   theme?: unknown;
   locale?: unknown;
   closeWindowBehavior?: unknown;
@@ -98,6 +99,12 @@ function readLocalUiSettings(): {
    * with no providers would drop the whole queue.
    */
   modelFailover: unknown;
+  /**
+   * Retry-error config is a local UI preference (not gateway-synced), so it
+   * lives in localStorage like chatRuntimeControls. Read raw; normalizeSettings
+   * validates preset codes and de-dupes custom patterns.
+   */
+  retryErrorSettings: unknown;
   theme: Theme;
   locale: Locale;
   closeWindowBehavior: CloseWindowBehavior;
@@ -137,6 +144,7 @@ function readLocalUiSettings(): {
         updates: defaults.updates,
         selectedModel: defaults.selectedModel,
         modelFailover: defaults.modelFailover,
+        retryErrorSettings: defaults.retryErrorSettings,
         theme: defaults.theme,
         locale: defaults.locale,
         closeWindowBehavior: defaults.closeWindowBehavior,
@@ -157,6 +165,7 @@ function readLocalUiSettings(): {
       updates: normalizeUpdateSettings(parsed?.updates ?? defaults.updates),
       selectedModel: normalizeSelectedModel(parsed?.selectedModel),
       modelFailover: parsed?.modelFailover ?? defaults.modelFailover,
+      retryErrorSettings: parsed?.retryErrorSettings ?? defaults.retryErrorSettings,
       theme: normalizeTheme(parsed?.theme ?? defaults.theme),
       locale: normalizeLocale(hasStoredLocale ? parsed?.locale : defaults.locale),
       closeWindowBehavior: normalizeCloseWindowBehavior(
@@ -171,6 +180,7 @@ function readLocalUiSettings(): {
       updates: defaults.updates,
       selectedModel: defaults.selectedModel,
       modelFailover: defaults.modelFailover,
+      retryErrorSettings: defaults.retryErrorSettings,
       theme: defaults.theme,
       locale: defaults.locale,
       closeWindowBehavior: defaults.closeWindowBehavior,
@@ -189,6 +199,7 @@ function writeLocalUiSettings(
     | "theme"
     | "locale"
     | "closeWindowBehavior"
+    | "retryErrorSettings"
   >,
 ) {
   const payload = {
@@ -200,6 +211,7 @@ function writeLocalUiSettings(
     theme: settings.theme,
     locale: settings.locale,
     closeWindowBehavior: settings.closeWindowBehavior,
+    retryErrorSettings: settings.retryErrorSettings,
   };
   localStorage.setItem(LOCAL_UI_SETTINGS_STORAGE_KEY, JSON.stringify(payload));
 }
@@ -265,6 +277,7 @@ export async function loadPersistedSettingsWithDefaults(): Promise<PersistedSett
     // the localStorage copy only migrates pre-SQLite installs forward.
     modelFailover: (persisted?.modelFailover ??
       localUi.modelFailover) as AppSettings["modelFailover"],
+    retryErrorSettings: localUi.retryErrorSettings as AppSettings["retryErrorSettings"],
     theme: localUi.theme,
     locale: localUi.locale,
     closeWindowBehavior: localUi.closeWindowBehavior,
@@ -385,7 +398,8 @@ export async function persistSettings(
     hasChanged(prev.selectedModel ?? null, next.selectedModel ?? null) ||
     hasChanged(prev.theme, next.theme) ||
     hasChanged(prev.locale, next.locale) ||
-    hasChanged(prev.closeWindowBehavior, next.closeWindowBehavior)
+    hasChanged(prev.closeWindowBehavior, next.closeWindowBehavior) ||
+    hasChanged(prev.retryErrorSettings, next.retryErrorSettings)
   ) {
     writeLocalUiSettings({
       skills: next.skills,
@@ -396,6 +410,7 @@ export async function persistSettings(
       theme: next.theme,
       locale: next.locale,
       closeWindowBehavior: next.closeWindowBehavior,
+      retryErrorSettings: next.retryErrorSettings,
     });
   }