diff --git a/README.md b/README.md index bfc8c706..e8bc0849 100644 --- a/README.md +++ b/README.md @@ -241,3 +241,4 @@ Read more about E2B on the [E2B website](https://e2b.dev) and the official [E2B - Next.js app with LLM + Code Interpreter and streaming - [TypeScript](./examples/nextjs-code-interpreter) - How to run a Docker container in E2B - [Python/TypeScript](./examples/docker-in-e2b) - How to run Playwright in E2B - [TypeScript](./examples/playwright-in-e2b) +- Self-healing Playwright tests with multi-model LLM fallback - [TypeScript](./examples/self-healing-playwright-tests) diff --git a/examples/self-healing-playwright-tests/.env.template b/examples/self-healing-playwright-tests/.env.template new file mode 100644 index 00000000..6f312bd4 --- /dev/null +++ b/examples/self-healing-playwright-tests/.env.template @@ -0,0 +1,21 @@ +# E2B sandbox API key — get one at https://e2b.dev +E2B_API_KEY= + +# At least one model provider key is required. +# The router will try them in the order they are configured (see ROUTER_ORDER). +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +GOOGLE_GENERATIVE_AI_API_KEY= + +# Comma-separated provider order. Default: anthropic,openai,google +# Providers without a key are skipped automatically. +ROUTER_ORDER=anthropic,openai,google + +# How many self-healing attempts before giving up. Default: 3 +HEAL_MAX_ATTEMPTS=3 + +# Optional: pin specific models per provider. +# Defaults are sensible for early-2026 (Opus 4.7, GPT-5, Gemini 2.5 Pro). +ANTHROPIC_MODEL=claude-opus-4-7 +OPENAI_MODEL=gpt-5 +GOOGLE_MODEL=gemini-2.5-pro diff --git a/examples/self-healing-playwright-tests/README.md b/examples/self-healing-playwright-tests/README.md new file mode 100644 index 00000000..22c67ebe --- /dev/null +++ b/examples/self-healing-playwright-tests/README.md @@ -0,0 +1,173 @@ +# Self-Healing Playwright Tests with Multi-Model Fallback + +Generate a Playwright test from a natural-language spec, run it inside an isolated E2B sandbox, and — when it fails — feed the page snapshot back to the LLM so it rewrites the test with more robust selectors. Multi-model fallback (Claude → GPT → Gemini) via the Vercel AI SDK keeps the pipeline alive if one provider is down. + +The pattern is inspired by the self-healing behaviour used in production at [qualitymax.io](https://qualitymax.io); this is a standalone TypeScript example contributed to the E2B Cookbook. + +## Why + +Browser tests break the moment a button moves or a class name changes. Two common responses: + +1. Brittle CSS selectors + a human on call — the status quo. +2. Self-healing tests — capture the page state at failure, hand it to an LLM with the original spec, get back a fixed test. + +(2) only works if you can run the regenerated test in a clean, isolated environment without contaminating CI. E2B sandboxes give you exactly that: throwaway Firecracker microVMs, ~200 ms cold-start on the pre-baked `playwright-chromium` template. + +## Architecture + +``` + TestSpec (NL description + URL) + │ + ▼ + ┌──────────────┐ fallback chain (Claude → GPT → Gemini) + │ router.ts │ via Vercel AI SDK + └──────┬───────┘ + │ generated Playwright TS + ▼ + ┌──────────────┐ fresh sandbox per attempt + │ runner.ts │ runs test in `playwright-chromium` template + └──────┬───────┘ + │ pass / fail + stdout / stderr + page snapshot + ▼ + ┌──────────────┐ on fail: rewrite prompt with snapshot + errors + │ healer.ts │ back through router → runner, up to N attempts + └──────────────┘ +``` + +Three small modules, ~250 lines total: + +- **`src/router.ts`** — multi-model router. Reads `ROUTER_ORDER` from env, drops providers without keys, falls back on any error. +- **`src/runner.ts`** — E2B execution. Spins up `Sandbox.create('playwright-chromium')`, writes the generated test, installs `@playwright/test` in a user-writable work dir, executes, captures result. +- **`src/healer.ts`** — failure → snapshot → prompt → retry loop. + +## Setup + +```bash +npm install +cp .env.template .env # then fill in keys +npm run typecheck # sanity check +npm run example:basic # run the simplest example +``` + +You need: + +- An [E2B](https://e2b.dev) API key (free tier is enough). +- At least one of: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_GENERATIVE_AI_API_KEY`. The router silently drops providers without keys, so a partial setup still works. + +Per attempt wall clock: **~30–80 s** (mostly LLM latency + test run). Full healing demo with 2–3 attempts: **~2–4 min**. For CI where you want the ~5–10 s `@playwright/test` install cost gone too, build your own E2B template on top of `playwright-chromium` with it pre-baked. + +## Examples + +| Script | What it shows | +| --- | --- | +| `examples/01-basic-test.ts` | Simplest path: spec → generated test → one sandboxed run, no healing. | +| `examples/02-self-healing.ts` | The headline. A vague spec produces a brittle test, the failure snapshot is fed back to the LLM, the retry passes. | +| `examples/03-multi-model-fallback.ts` | Simulate a provider outage by poisoning the Anthropic key. The router falls back to OpenAI / Google. Needs at least two provider keys configured to demonstrate anything meaningful. | +| `examples/04-parallel-suite.ts` | Run multiple healing tests concurrently in isolated sandboxes. | + +Run any of them with `npm run example:` (see `package.json`). + +## How healing actually works + +The non-obvious bit is what we feed back to the LLM after a failure: + +1. **The runner injects a `test.afterEach` capture hook** into the generated test before each attempt (sentinel-bracketed, stripped and reattached on every heal). On any failed test it writes `await page.content()` to a known path inside the sandbox. The LLM never has to remember to do this — see "Battle-tested patterns" below for why that matters. +2. **The runner** reads that snapshot off the sandbox filesystem after the failed run. +3. **The healer classifies the failure** (`strict_mode_violation`, `locator_not_found`, `timeout`, `assertion_failed`, `unknown`) and steers the prompt accordingly. Strict-mode violations in particular get a dedicated hint pointing at three concrete fix patterns instead of letting the LLM re-emit equivalent fragile locators. +4. **The heal prompt** includes the previous code, the truncated stdout/stderr, the page snapshot, and the failure-type-specific hint. +5. **Each retry** spins up a fresh sandbox, so failures don't compound. + +`HEAL_MAX_ATTEMPTS` caps the loop (default `3`). + +### What a real run looks like + +``` +$ npm run example:healing + +→ attempt 1: LLM writes getByRole('link', { name: /Docs|Documentation/i }) + ✘ strict mode violation: locator resolved to 4 elements + → healer reads /app/failure.html, feeds DOM back to the LLM +→ attempt 2: LLM rewrites with a more specific selector + ✓ test passes + +Final: ✓ passed +Attempts: 2 + attempt 1 -- fail (generated by google) + attempt 2 -- pass (generated by google) +``` + +Each attempt spins up a fresh sandbox — failures from attempt 1 can't contaminate attempt 2. + +## Battle-tested patterns + +The first cut of this example trusted the LLM to do more of the wiring. Three problems showed up in production at [qualitymax.io](https://qualitymax.io) and got rolled back into the code here. If you copy this pattern into your own agent, these are the ones worth keeping: + +### 1. Inject the page-snapshot hook yourself; don't ask the LLM to write it + +Original approach: tell the LLM to call `fs.writeFileSync(somePath, await page.content())` in its own `afterEach`. Three failure modes, none loud: + +- LLM omits the hook in the heal pass → no snapshot → healing degrades to guessing from `stderr`. +- LLM writes to a root-owned path (`/app/...`) → `EACCES` → silent, the test still fails for the original reason but you have no DOM. +- LLM duplicates `import fs from 'fs'` on heal → `SyntaxError`, every retry now fails before the test runs. + +Fix: `src/runner.ts` injects a controlled `test.afterEach` block between sentinels (`// --- begin self-healing capture hook ---` / `// --- end ---`) and strip-and-reattaches it on every heal. The hook uses `require('fs')` inline instead of a top-level import so a duplicated injection can't break the file. + +### 2. Classify the failure before prompting + +`src/runner.ts → classifyFailure()` buckets failures into one of five types. The healer only injects the strict-mode hint when `strict_mode_violation` matched. Without classification, the strict-mode guidance pollutes prompts for unrelated failures (timeouts, assertion mismatches) and pushes the LLM toward "fixing" non-issues. + +Strict-mode-violation handling is the highest-leverage instance: without the explicit hint, the LLM kept swapping `.first()` in or trying a different `[class*=…]` matcher and burning the whole heal budget on the same failure. With the hint pointing at "anchor by unique text and walk up" / "scope by parent role" / "use the exact `data-test` from the snapshot", it usually heals in one extra attempt. + +### 3. Sentinel-based progress, not log scraping + +`src/runner.ts` emits `QMAX_PHASE:` markers at known points in the in-sandbox command and forwards them via the optional `onProgress` callback. A grep for a sentinel survives the npm/Playwright version bump that would break a regex over their normal log output. The pattern reads from `commands.run({ onStdout, onStderr })` rather than waiting for the run to finish, so UI consumers see progress in real time. + +```ts +const result = await runInSandbox(generated, { + onProgress: (phase) => console.log(`[${phase}]`), +}); +``` + +If you want a *live browser feed* on top of progress markers (watch the test drive Chromium while it runs), see the [playwright-live-vnc-feed](../playwright-live-vnc-feed) example. + +## Why multi-model fallback matters + +LLM provider availability is not 100%. On 15 April 2026 an Anthropic incident took down Claude API access for several hours. Single-provider agents went dark; anything with a router survived. + +`examples/03-multi-model-fallback.ts` reproduces the failure mode locally by poisoning the Anthropic key. You need at least two provider keys configured for this to show anything meaningful — with only one key the router has nowhere to fall back to. Output on a two-key setup: + +``` +→ Anthropic key is poisoned. Router should fall back. +✓ test generated -- the router landed on openai +✓ test passed +``` + +Production code will want to: +- Distinguish transient (5xx, rate limit) from permanent (auth) errors and retry transients on the same provider before falling back. +- Track provider-level success rates and reorder dynamically. +- Cap total wall-clock so a slow fallback doesn't block the loop. + +The pattern here is the floor, not the ceiling. + +## Notes on the `playwright-chromium` template + +The template ships `playwright` + browser binaries pre-installed at `/app`, but not `@playwright/test` (the test runner). The runner handles this: + +- `/app` is root-owned, so test files and additional npm installs go to `/home/user/work` instead. +- `@playwright/test` is installed there, pinned to `1.51.1` to match the template's bundled Playwright version (browser/runner version skew causes "chrome-headless-shell-: cannot execute binary" failures). +- `PLAYWRIGHT_BROWSERS_PATH` points at `/app/node_modules/playwright-core/.local-browsers` so the test runner re-uses the template's Chromium instead of downloading its own. + +All three bits are documented inline in `src/runner.ts`. + +## Adapting it + +- **Use a different test framework.** Replace the `ensureTestRunner` step in `src/runner.ts` and the `SYSTEM_RULES` in `src/healer.ts`. Cypress, Vitest browser mode, Pytest + Selenium — anything that runs in a Linux sandbox works. +- **Add more providers.** `src/router.ts` maps provider names to AI SDK clients. Add a case to `modelFor()` and an entry to `ENV_KEY_BY_PROVIDER`. +- **Build a custom E2B template.** Pre-bake `@playwright/test` on top of `playwright-chromium` to drop each attempt back toward the 200 ms cold-start floor. +- **Wire it into CI.** The whole entry point is `runHealingTest(spec, maxAttempts)`. Call it from a GitHub Action and post the trace to your build summary. + +## Credits + +- Self-healing pattern inspired by [qualitymax.io](https://qualitymax.io). +- Sandbox runtime: [E2B](https://e2b.dev). +- Model orchestration: [Vercel AI SDK](https://sdk.vercel.ai). diff --git a/examples/self-healing-playwright-tests/examples/01-basic-test.ts b/examples/self-healing-playwright-tests/examples/01-basic-test.ts new file mode 100644 index 00000000..2df11b71 --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/01-basic-test.ts @@ -0,0 +1,53 @@ +/** + * Example 1 -- the simplest path through the system. + * + * Generate a Playwright test from a one-line natural-language spec + * and run it once in a fresh E2B sandbox. No healing. + * + * Run: pnpm tsx examples/01-basic-test.ts + */ + +import 'dotenv/config'; + +import { route } from '../src/router.js'; +import { runInSandbox } from '../src/runner.js'; +import type { GeneratedTest, TestSpec } from '../src/types.js'; + +const spec: TestSpec = { + url: 'https://e2b.dev', + description: + 'The page should mention "AI agents" somewhere on it within 10 seconds of loading.', +}; + +const PROMPT = `You are an expert Playwright test author. +Output a single TypeScript file -- no markdown fences, no commentary. +Use @playwright/test, import { test, expect }. + +Spec: +URL: ${spec.url} +What to verify: ${spec.description} + +Return only the TypeScript file content.`; + +async function main(): Promise { + console.log('→ generating test...'); + const { text, provider } = await route(PROMPT); + const generated: GeneratedTest = { code: text.trim(), provider }; + console.log(`✓ generated by ${provider} (${generated.code.length} chars)`); + + console.log('→ running in E2B playwright-chromium sandbox (~30-50s)...'); + const result = await runInSandbox(generated); + console.log(result.passed ? '✓ test passed' : '✗ test failed'); + if (!result.passed) { + console.log('--- stdout ---'); + console.log(result.stdout); + console.log('--- stderr ---'); + console.log(result.stderr); + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/examples/02-self-healing.ts b/examples/self-healing-playwright-tests/examples/02-self-healing.ts new file mode 100644 index 00000000..85b2850e --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/02-self-healing.ts @@ -0,0 +1,51 @@ +/** + * Example 2 -- the headline example. + * + * Run a test that is *deliberately likely to fail on the first try* + * (description is vague enough that the LLM may pick brittle selectors). + * The healing loop kicks in: failure stdout + a captured page snapshot + * are fed back into the LLM, which rewrites the test with more robust + * selectors. We retry up to HEAL_MAX_ATTEMPTS times. + * + * This is the qualitymax.io self-healing pattern, ported to E2B. + * + * Run: pnpm tsx examples/02-self-healing.ts + */ + +import 'dotenv/config'; + +import { runHealingTest } from '../src/healer.js'; +import type { TestSpec } from '../src/types.js'; + +const spec: TestSpec = { + url: 'https://e2b.dev', + description: + 'Click whatever link or button appears to lead to the documentation, ' + + 'then verify the destination page contains the word "sandbox".', + context: + 'The site nav may have changed since the LLM was trained. Prefer ' + + 'role / accessible-name / data-testid based selectors.', +}; + +async function main(): Promise { + const trace = await runHealingTest(spec, 3); + + console.log(`\nFinal: ${trace.finalPassed ? '✓ passed' : '✗ failed'}`); + console.log(`Attempts: ${trace.attempts.length}`); + + trace.attempts.forEach((a) => { + console.log( + ` attempt ${a.attempt} -- ${a.result.passed ? 'pass' : 'fail'} ` + + `(generated by ${a.generated.provider})`, + ); + }); + + if (!trace.finalPassed) { + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts b/examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts new file mode 100644 index 00000000..640bb982 --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts @@ -0,0 +1,52 @@ +/** + * Example 3 -- show the router fall back from a "broken" provider. + * + * We simulate the Anthropic outage of 15 April 2026 by overriding the + * Anthropic key with a guaranteed-bad value. The router should detect + * the auth failure and move on to OpenAI (or Google). The test still + * gets generated and run. + * + * This is the production behavior that kept qualitymax.io serving + * customers during that incident -- the architecture is the same here. + * + * Run: pnpm tsx examples/03-multi-model-fallback.ts + */ + +import 'dotenv/config'; + +import { route } from '../src/router.js'; +import { runInSandbox } from '../src/runner.js'; +import type { GeneratedTest } from '../src/types.js'; + +// Simulate the outage by poisoning Anthropic auth before the SDK reads it. +// We keep OpenAI / Google keys real so the fallback chain has somewhere +// to go. +process.env.ANTHROPIC_API_KEY = 'sk-ant-deliberately-invalid-for-demo'; + +const PROMPT = `You are an expert Playwright test author. +Output a single TypeScript file -- no markdown fences, no commentary. +Use @playwright/test, import { test, expect }. + +Spec: +URL: https://e2b.dev +What to verify: page is non-empty. + +Return only the TypeScript file content.`; + +async function main(): Promise<void> { + console.log('→ Anthropic key is poisoned. Router should fall back.'); + const { text, provider } = await route(PROMPT); + console.log(`✓ test generated -- the router landed on ${provider}`); + + const generated: GeneratedTest = { code: text.trim(), provider }; + const result = await runInSandbox(generated); + console.log(result.passed ? '✓ test passed' : '✗ test failed'); + if (!result.passed) { + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/examples/04-parallel-suite.ts b/examples/self-healing-playwright-tests/examples/04-parallel-suite.ts new file mode 100644 index 00000000..092fe7ad --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/04-parallel-suite.ts @@ -0,0 +1,50 @@ +/** + * Example 4 -- run multiple healing tests in parallel sandboxes. + * + * Each TestSpec gets its own isolated E2B sandbox so failures and + * state don't leak between them. This is roughly how a real test + * suite would be wired up against E2B in production. + * + * Run: pnpm tsx examples/04-parallel-suite.ts + */ + +import 'dotenv/config'; + +import { runHealingTest } from '../src/healer.js'; +import type { TestSpec } from '../src/types.js'; + +const suite: TestSpec[] = [ + { + url: 'https://e2b.dev', + description: 'The page should mention "AI agents" somewhere on it.', + }, + { + url: 'https://e2b.dev/docs', + description: 'There should be a link to the Cookbook somewhere on the page.', + }, + { + url: 'https://e2b.dev/blog', + description: 'The page should list at least one blog post title.', + }, +]; + +async function main(): Promise<void> { + const traces = await Promise.all(suite.map((spec) => runHealingTest(spec, 2))); + + let failed = 0; + traces.forEach((trace) => { + const status = trace.finalPassed ? '✓' : '✗'; + console.log(`${status} ${trace.spec.url} (${trace.attempts.length} attempts)`); + if (!trace.finalPassed) failed++; + }); + + console.log(`\n${suite.length - failed}/${suite.length} passed`); + if (failed > 0) { + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/package-lock.json b/examples/self-healing-playwright-tests/package-lock.json new file mode 100644 index 00000000..c849badb --- /dev/null +++ b/examples/self-healing-playwright-tests/package-lock.json @@ -0,0 +1,991 @@ +{ + "name": "self-healing-playwright-tests", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "self-healing-playwright-tests", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/anthropic": "^1.2.0", + "@ai-sdk/google": "^1.2.0", + "@ai-sdk/openai": "^1.2.0", + "@e2b/code-interpreter": "^1.5.0", + "ai": "^4.3.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@ai-sdk/anthropic": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-1.2.12.tgz", + "integrity": "sha512-YSzjlko7JvuiyQFmI9RN1tNZdEiZxc+6xld/0tq/VkJaHpEzGAb1yiNxxvmYVcjvfu/PcvCxAAYXmTYQQ63IHQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/google": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-1.2.22.tgz", + "integrity": "sha512-Ppxu3DIieF1G9pyQ5O1Z646GYR0gkC57YdBqXJ82qvCdhEhZHu0TWhmnOoeIWe2olSbuDeoOY+MfJrW8dzS3Hw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "1.3.24", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-1.3.24.tgz", + "integrity": "sha512-GYXnGJTHRTZc4gJMSmFRgEQudjqd4PUN0ZjQhPwOAYH1yOAvQoG/Ikqs+HyISRbLPCrhbZnPKCNHuRU4OfpW0Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", + "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.11.0.tgz", + "integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@connectrpc/connect": { + "version": "2.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-2.0.0-rc.3.tgz", + "integrity": "sha512-ARBt64yEyKbanyRETTjcjJuHr2YXorzQo0etyS5+P6oSeW8xEuzajA9g+zDnMcj1hlX2dQE93foIWQGfpru7gQ==", + "license": "Apache-2.0", + "peerDependencies": { + "@bufbuild/protobuf": "^2.2.0" + } + }, + "node_modules/@connectrpc/connect-web": { + "version": "2.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@connectrpc/connect-web/-/connect-web-2.0.0-rc.3.tgz", + "integrity": "sha512-w88P8Lsn5CCsA7MFRl2e6oLY4J/5toiNtJns/YJrlyQaWOy3RO8pDgkz+iIkG98RPMhj2thuBvsd3Cn4DKKCkw==", + "license": "Apache-2.0", + "peerDependencies": { + "@bufbuild/protobuf": "^2.2.0", + "@connectrpc/connect": "2.0.0-rc.3" + } + }, + "node_modules/@e2b/code-interpreter": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@e2b/code-interpreter/-/code-interpreter-1.5.1.tgz", + "integrity": "sha512-mkyKjAW2KN5Yt0R1I+1lbH3lo+W/g/1+C2lnwlitXk5wqi/g94SEO41XKdmDf5WWpKG3mnxWDR5d6S/lyjmMEw==", + "license": "MIT", + "dependencies": { + "e2b": "^1.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", + "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/ai": { + "version": "4.3.19", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.19.tgz", + "integrity": "sha512-dIE2bfNpqHN3r6IINp9znguYdhIOheKW2LDigAMrgt/upT3B8eBGPSCblENvaZGoq+hxaN9fSMzjWpbqloP+7Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/compare-versions": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.1.1.tgz", + "integrity": "sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==", + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "license": "Apache-2.0" + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/e2b": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/e2b/-/e2b-1.13.2.tgz", + "integrity": "sha512-m8acE/MzMAJo1A57DakR2X1Sl5Mt1tcQO2aJfygNaQHLXby/4xsjF0UeJUB70jF7xntiR41pAMbZEHnkzrT9tw==", + "license": "MIT", + "dependencies": { + "@bufbuild/protobuf": "^2.6.2", + "@connectrpc/connect": "2.0.0-rc.3", + "@connectrpc/connect-web": "2.0.0-rc.3", + "compare-versions": "^6.1.0", + "openapi-fetch": "^0.9.7", + "platform": "^1.3.6" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/jsondiffpatch": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz", + "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", + "license": "MIT", + "dependencies": { + "@types/diff-match-patch": "^1.0.36", + "chalk": "^5.3.0", + "diff-match-patch": "^1.0.5" + }, + "bin": { + "jsondiffpatch": "bin/jsondiffpatch.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/openapi-fetch": { + "version": "0.9.8", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.9.8.tgz", + "integrity": "sha512-zM6elH0EZStD/gSiNlcPrzXcVQ/pZo3BDvC6CDwRDUt1dDzxlshpmQnpD6cZaJ39THaSmwVCxxRrPKNM1hHrDg==", + "license": "MIT", + "dependencies": { + "openapi-typescript-helpers": "^0.0.8" + } + }, + "node_modules/openapi-typescript-helpers": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.0.8.tgz", + "integrity": "sha512-1eNjQtbfNi5Z/kFhagDIaIRj6qqDzhjNJKz8cmMW0CVdGwT6e1GLbAfgI0d28VTJa1A8jz82jm/4dG8qNoNS8g==", + "license": "MIT" + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, + "node_modules/react": { + "version": "19.2.5", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", + "integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, + "node_modules/swr": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.4.1.tgz", + "integrity": "sha512-2CC6CiKQtEwaEeNiqWTAw9PGykW8SR5zZX8MZk6TeAvEAnVS7Visz8WzphqgtQ8v2xz/4Q5K+j+SeMaKXeeQIA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/examples/self-healing-playwright-tests/package.json b/examples/self-healing-playwright-tests/package.json new file mode 100644 index 00000000..f1a6f18a --- /dev/null +++ b/examples/self-healing-playwright-tests/package.json @@ -0,0 +1,46 @@ +{ + "name": "self-healing-playwright-tests", + "version": "0.1.0", + "description": "Cookbook example: self-healing browser tests on E2B sandboxes with Vercel AI SDK and multi-model fallback (Claude / GPT / Gemini).", + "license": "Apache-2.0", + "author": "Ruslan Strazhnyk <strazhnyk@gmail.com>", + "type": "module", + "scripts": { + "example:basic": "tsx examples/01-basic-test.ts", + "example:healing": "tsx examples/02-self-healing.ts", + "example:fallback": "tsx examples/03-multi-model-fallback.ts", + "example:suite": "tsx examples/04-parallel-suite.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@ai-sdk/anthropic": "^1.2.0", + "@ai-sdk/google": "^1.2.0", + "@ai-sdk/openai": "^1.2.0", + "@e2b/code-interpreter": "^1.5.0", + "ai": "^4.3.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + }, + "keywords": [ + "e2b", + "ai-sdk", + "vercel-ai-sdk", + "playwright", + "self-healing-tests", + "ai-agents", + "multi-model", + "claude", + "gpt", + "gemini", + "cookbook", + "sandbox" + ] +} diff --git a/examples/self-healing-playwright-tests/src/healer.ts b/examples/self-healing-playwright-tests/src/healer.ts new file mode 100644 index 00000000..3a5824bd --- /dev/null +++ b/examples/self-healing-playwright-tests/src/healer.ts @@ -0,0 +1,173 @@ +/** + * Healing loop: generate test → run → on failure, ask LLM to fix it + * using the actual error output and a captured page snapshot, retry. + * + * Self-healing is the qualitymax.io feature this example demonstrates + * on E2B's substrate. The pattern is small enough to copy into any + * agent codebase that needs to keep brittle external tests running + * across UI changes. + */ + +import { + type FailureType, + type GeneratedTest, + type HealAttempt, + type HealingTrace, + type TestSpec, +} from './types.js'; +import { route } from './router.js'; +import { runInSandbox } from './runner.js'; + +const DEFAULT_MAX_ATTEMPTS = Number.parseInt( + process.env.HEAL_MAX_ATTEMPTS ?? '3', + 10, +); + +/** + * Instructions the LLM gets for both first-generation and healing. + * + * Note what we *don't* ask for: the LLM no longer has to remember to + * dump page.content() on failure. The runner injects a controlled + * `test.afterEach` hook for that (see `injectCaptureHook` in runner.ts). + * Asking the LLM to do it produced three battle-tested failure modes + * at qualitymax.io — see the hook's inline comment for the full story. + * + * We also forbid markdown fences so the runner can pipe the text + * straight into a .ts file. Gemini in particular still occasionally + * wraps output in fences; the router has a defensive `stripCodeFences` + * pass for that, but the prompt belt-and-braces it. + */ +const SYSTEM_RULES = `You are an expert Playwright test author. +Output a single TypeScript file -- no markdown fences, no commentary. +The file must use @playwright/test and import { test, expect }. +Prefer robust selectors: getByRole / getByText / data-testid / data-test. +When multiple elements could match a locator, anchor by unique text and +walk up to a container, or scope by parent role. Never use [class*=...] +substring matchers -- they trigger strict-mode violations on real pages.`; + +function buildGeneratePrompt(spec: TestSpec): string { + return `${SYSTEM_RULES} + +Write a Playwright test for the following spec: + +URL: ${spec.url} +What to verify: ${spec.description} +${spec.context ? `Extra context: ${spec.context}` : ''} + +Return only the TypeScript file content.`; +} + +/** + * Hint appended to the heal prompt for strict-mode violations. + * + * Battle-tested at qualitymax.io: without this, the healer treats a + * "resolved to 4 elements" error the same as "resolved to 0 elements" + * and frequently re-emits an equivalent fragile locator (e.g. swaps + * `.first()` in, swaps another `[class*=…]` in). Burns the whole heal + * budget on the same failure. Steering the LLM to one of three concrete + * patterns turned the strict-mode failure mode from "blocks healing" + * into "heals in one extra attempt" in our internal metrics. + */ +const STRICT_MODE_HINT = ` +=== STRICT-MODE VIOLATION DETECTED === +The failing locator matches multiple elements. Playwright's strict mode +rejects ambiguous locators. Do NOT add \`.first()\` blindly -- almost always +the right fix is one of: + 1. Anchor by unique text and walk up to the container: + page.getByText('Total Balance').locator('..').getByText(/\\$\\d+/) + 2. Scope by parent role/region: + page.getByRole('region', { name: 'Total Balance' }).getByText(/\\$\\d+/) + 3. Use the exact data-test attribute if one is in the page snapshot: + page.locator('[data-test="total-balance"]') +Substring class matchers like [class*="balance"] are forbidden -- they always +hit multiple elements on a real page. Replace them with one of the above.`; + +function buildHealPrompt( + spec: TestSpec, + previous: GeneratedTest, + failure: { + stdout: string; + stderr: string; + snapshot?: string; + failureType?: FailureType; + }, +): string { + const snapshotBlock = failure.snapshot + ? `\nPage HTML at failure (truncated to 8000 chars):\n${failure.snapshot.slice(0, 8000)}\n` + : '\nNo page snapshot was captured.\n'; + + // Only inject the strict-mode hint when the failure is actually a + // strict-mode violation. Including it unconditionally pollutes prompts + // for unrelated failures (assertion mismatches, timeouts) and pushes + // the LLM toward "fixing" non-issues. + const failureHint = failure.failureType === 'strict_mode_violation' ? STRICT_MODE_HINT : ''; + + return `${SYSTEM_RULES} + +The previous test below failed. Fix it. + +Spec: +URL: ${spec.url} +What to verify: ${spec.description} +${spec.context ? `Extra context: ${spec.context}` : ''} + +Previous test: +${previous.code} + +Failure stdout: +${failure.stdout.slice(0, 4000)} + +Failure stderr: +${failure.stderr.slice(0, 2000)} +${failureHint} +${snapshotBlock} +Return only the new TypeScript file content.`; +} + +async function generate(spec: TestSpec): Promise<GeneratedTest> { + const { text, provider, usage } = await route(buildGeneratePrompt(spec)); + return { code: text.trim(), provider, usage }; +} + +async function heal( + spec: TestSpec, + previous: HealAttempt, +): Promise<GeneratedTest> { + const { text, provider, usage } = await route( + buildHealPrompt(spec, previous.generated, { + stdout: previous.result.stdout, + stderr: previous.result.stderr, + snapshot: previous.result.failureSnapshot, + failureType: previous.result.failureType, + }), + ); + return { code: text.trim(), provider, usage }; +} + +/** + * Drive a test from natural-language spec to passing run, healing on + * each failure. Returns the full attempt log so the caller can inspect + * which provider produced which version. + */ +export async function runHealingTest( + spec: TestSpec, + maxAttempts: number = DEFAULT_MAX_ATTEMPTS, +): Promise<HealingTrace> { + const attempts: HealAttempt[] = []; + let generated = await generate(spec); + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const result = await runInSandbox(generated); + attempts.push({ attempt, generated, result }); + + if (result.passed) { + return { spec, attempts, finalPassed: true }; + } + if (attempt === maxAttempts) { + break; + } + generated = await heal(spec, attempts[attempts.length - 1]!); + } + + return { spec, attempts, finalPassed: false }; +} diff --git a/examples/self-healing-playwright-tests/src/index.ts b/examples/self-healing-playwright-tests/src/index.ts new file mode 100644 index 00000000..9c07e8ab --- /dev/null +++ b/examples/self-healing-playwright-tests/src/index.ts @@ -0,0 +1,21 @@ +/** + * Public entry points for the cookbook example. + */ + +export { runHealingTest } from './healer.js'; +export { route, loadRouterConfig } from './router.js'; +export { classifyFailure, runInSandbox } from './runner.js'; +export type { RunOptions } from './runner.js'; + +export type { + FailureType, + GeneratedTest, + HealAttempt, + HealingTrace, + ProviderName, + RouterConfig, + RunPhase, + RunResult, + TestSpec, +} from './types.js'; +export { AllProvidersFailedError } from './types.js'; diff --git a/examples/self-healing-playwright-tests/src/router.ts b/examples/self-healing-playwright-tests/src/router.ts new file mode 100644 index 00000000..cbcdbaec --- /dev/null +++ b/examples/self-healing-playwright-tests/src/router.ts @@ -0,0 +1,135 @@ +/** + * Multi-model router with fallback chain. + * + * Tries providers in order. On rate-limit, 5xx, or network error, + * falls back to the next configured provider. Throws AllProvidersFailedError + * if every provider fails for the same call. + * + * This is the same pattern qualitymax.io uses to survive single-provider + * outages (the platform stayed up through the documented Anthropic + * outage of 15 April 2026 because the router fell back to GPT and Gemini). + */ + +import { generateText } from 'ai'; +import type { LanguageModelV1 } from 'ai'; +import { anthropic } from '@ai-sdk/anthropic'; +import { openai } from '@ai-sdk/openai'; +import { google } from '@ai-sdk/google'; + +import { + AllProvidersFailedError, + type ProviderName, + type RouterConfig, +} from './types.js'; + +const DEFAULT_MODELS: Record<ProviderName, string> = { + anthropic: process.env.ANTHROPIC_MODEL ?? 'claude-opus-4-7', + openai: process.env.OPENAI_MODEL ?? 'gpt-5', + google: process.env.GOOGLE_MODEL ?? 'gemini-2.5-pro', +}; + +const ENV_KEY_BY_PROVIDER: Record<ProviderName, string> = { + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + google: 'GOOGLE_GENERATIVE_AI_API_KEY', +}; + +function modelFor(provider: ProviderName, override?: string): LanguageModelV1 { + const id = override ?? DEFAULT_MODELS[provider]; + switch (provider) { + case 'anthropic': + return anthropic(id); + case 'openai': + return openai(id); + case 'google': + return google(id); + } +} + +/** + * Read the router config from env vars, with sensible defaults. + * + * Providers without a configured API key are silently dropped from the + * order so that a partial setup still works. + */ +export function loadRouterConfig(): RouterConfig { + const raw = process.env.ROUTER_ORDER ?? 'anthropic,openai,google'; + const requested = raw + .split(',') + .map((s) => s.trim().toLowerCase()) + .filter((s): s is ProviderName => + s === 'anthropic' || s === 'openai' || s === 'google', + ); + const order = requested.filter((p) => Boolean(process.env[ENV_KEY_BY_PROVIDER[p]])); + if (order.length === 0) { + throw new Error( + 'No usable provider keys found. Set at least one of ' + + `${Object.values(ENV_KEY_BY_PROVIDER).join(', ')} in your environment.`, + ); + } + return { order }; +} + +export interface RouteCallResult { + text: string; + provider: ProviderName; + usage?: { promptTokens?: number; completionTokens?: number }; +} + +/** + * Strip a single outer markdown fence if present. + * + * LLMs occasionally wrap their response in ```lang ... ``` despite + * prompts that explicitly forbid it (Gemini is the usual offender). + * Since every caller in this cookbook expects raw TypeScript that can + * be written straight to a .ts file, we normalise at the router. + * + * Surgical by design: only strips when the trimmed text starts AND + * ends with a fence, so response bodies that legitimately contain + * inline fences are left alone. + */ +function stripCodeFences(text: string): string { + const trimmed = text.trim(); + const match = trimmed.match(/^```[^\n]*\n([\s\S]*?)\n?```$/); + return match ? match[1]! : trimmed; +} + +/** + * Try the configured providers in order until one returns a response. + * + * The error classification here is intentionally conservative -- on any + * thrown error we move on to the next provider. Production code may want + * to retry transient errors on the same provider before falling back. + */ +export async function route( + prompt: string, + config: RouterConfig = loadRouterConfig(), +): Promise<RouteCallResult> { + const failures: { provider: ProviderName; error: Error }[] = []; + + for (const provider of config.order) { + try { + const model = modelFor(provider, config.models?.[provider]); + const { text, usage } = await generateText({ + model, + prompt, + // Slightly lower temperature to keep generated test code stable. + temperature: 0.2, + }); + return { + text: stripCodeFences(text), + provider, + usage: { + promptTokens: usage?.promptTokens, + completionTokens: usage?.completionTokens, + }, + }; + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + failures.push({ provider, error: e }); + // Loop continues to the next provider in `order`. + } + } + + throw new AllProvidersFailedError(failures); +} diff --git a/examples/self-healing-playwright-tests/src/runner.ts b/examples/self-healing-playwright-tests/src/runner.ts new file mode 100644 index 00000000..9151d388 --- /dev/null +++ b/examples/self-healing-playwright-tests/src/runner.ts @@ -0,0 +1,297 @@ +/** + * E2B sandbox runner: writes a generated Playwright test file into a + * fresh sandbox on E2B's `playwright-chromium` template (Playwright + + * Chromium + system deps pre-baked), executes the test, and captures + * pass/fail plus stdout/stderr. + * + * Each call spins up a new sandbox so failures are isolated. Cold start + * is ~200ms on the pre-baked template; swap to `base` and add an + * `installPlaywright` step if you need to demonstrate bring-your-own. + */ + +import { CommandExitError, Sandbox } from '@e2b/code-interpreter'; + +import type { GeneratedTest, RunPhase, RunResult } from './types.js'; + +const TEMPLATE = 'playwright-chromium'; + +// Two-directory layout: the template's /app is owned by root and +// contains the pre-baked Playwright browser binaries, so we can't +// write our test files or install @playwright/test there. /home/user +// is user-writable; we keep tests and the test-runner package here and +// point PLAYWRIGHT_BROWSERS_PATH at /app's browser cache. +const WORK_DIR = '/home/user/work'; +const BROWSERS_PATH = '/app/node_modules/playwright-core/.local-browsers'; +const TEST_FILE_PATH = `${WORK_DIR}/test.spec.ts`; +const SNAPSHOT_PATH = `${WORK_DIR}/failure.html`; + +// Pin @playwright/test to match the version of `playwright` bundled in +// the template (check with `cat /app/package.json` in a probe sandbox). +// Version skew between the test runner and the browser binaries causes +// "chrome-headless-shell-<rev>: cannot execute binary" type failures. +const PLAYWRIGHT_VERSION = '1.51.1'; + +// Sentinels used to bracket the injected page-snapshot hook so we can +// strip-and-reattach it on every heal pass. Without strip-and-reattach, +// successive heals either: +// (a) drop the hook entirely → no snapshot → healing degrades to +// guessing from stderr; or +// (b) duplicate the hook → "Identifier '_fs' has already been +// declared" SyntaxError → every retry fails before the test runs. +// Battle-tested at qualitymax.io after both modes bit us in production. +const HOOK_BEGIN = '// --- begin self-healing capture hook ---'; +const HOOK_END = '// --- end self-healing capture hook ---'; + +// We inline `require('fs')` inside the callback rather than adding a +// top-level `import fs from 'fs'`. Two reasons: +// 1. The LLM-generated code may or may not already import fs — adding +// our own import risks a duplicate binding. +// 2. If the LLM accidentally pastes the hook twice during a heal, two +// top-level `_fs` bindings produce a SyntaxError that's invisible +// until the next run. `require()` is cached and binding-free, so +// duplication is harmless (the strip-and-reattach still runs). +const CAPTURE_HOOK = `${HOOK_BEGIN} +test.afterEach(async ({ page }, testInfo) => { + if (testInfo.status !== testInfo.expectedStatus) { + try { + const _html = await page.content(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + require('fs').writeFileSync('${SNAPSHOT_PATH}', _html.substring(0, 60000)); + } catch (_e) { + // Best-effort: a snapshot failure must never mask the real test failure. + } + } +}); +${HOOK_END}`; + +const HOOK_STRIP_RE = new RegExp( + `\\n*${HOOK_BEGIN.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}[\\s\\S]*?${HOOK_END.replace( + /[.*+?^${}()|[\\]\\\\]/g, + '\\\\$&', + )}\\n*`, + 'g', +); + +/** + * Remove a previously injected capture hook (idempotent). + * Always strip then reattach on each heal — see HOOK_BEGIN comment. + */ +function stripCaptureHook(code: string): string { + return code.includes(HOOK_BEGIN) ? code.replace(HOOK_STRIP_RE, '\n') : code; +} + +/** + * Append our controlled capture hook to the LLM-generated test. + * + * We do this server-side instead of asking the LLM to write the snapshot + * itself (the obvious approach). Battle-tested reason: when the LLM is + * responsible for the snapshot path, you accumulate three failure modes + * none of which fail loudly: + * + * 1. LLM forgets the hook in the heal pass. + * 2. LLM writes to a root-owned directory (`/app/...`) → EACCES, + * silent: the test fails for unrelated reasons and you have no DOM. + * 3. LLM duplicates `import fs from 'fs'` on heal → SyntaxError. + * + * Injecting the hook ourselves makes the snapshot a guaranteed side-effect + * of any failure, not a thing the agent might or might not remember to do. + */ +function injectCaptureHook(code: string): string { + return `${stripCaptureHook(code).trimEnd()}\n\n${CAPTURE_HOOK}\n`; +} + +export const _internals = { stripCaptureHook, injectCaptureHook, CAPTURE_HOOK }; + +/** + * Write the generated test into the sandbox along with a small Playwright + * config that drops the test runner into the working directory. + */ +async function prepareSandbox(sandbox: Sandbox, code: string): Promise<void> { + await sandbox.files.write(TEST_FILE_PATH, injectCaptureHook(code)); + await sandbox.files.write( + `${WORK_DIR}/playwright.config.ts`, + `import { defineConfig } from '@playwright/test'; +export default defineConfig({ + testDir: '${WORK_DIR}', + reporter: 'list', + use: { headless: true }, +}); +`, + ); +} + +/** + * The `playwright-chromium` template ships with `playwright` + browser + * binaries pre-installed at /app, but not `@playwright/test` (the test + * runner). Create a user-writable work dir and install just that -- + * cheap (~5-10s) compared to a full browser install. + * + * If E2B later adds `@playwright/test` to the template at a location + * we can resolve from, this function can be deleted. + */ +async function ensureTestRunner(sandbox: Sandbox): Promise<void> { + await sandbox.commands.run( + `mkdir -p ${WORK_DIR} && ` + + `cd ${WORK_DIR} && ` + + `npm init -y >/dev/null && ` + + `npm install --no-audit --no-fund --silent @playwright/test@${PLAYWRIGHT_VERSION}`, + { timeoutMs: 120 * 1000 }, + ); +} + +/** + * Best-effort read of the page snapshot the injected hook writes on failure. + * + * We control where the snapshot lives (`SNAPSHOT_PATH`) and we control the + * hook that writes it, so the only reasons the file is missing are: + * - the test crashed before page.content() resolved (e.g. browser launch + * failure); or + * - the failing fixture was so broken Playwright never reached afterEach. + * In both cases we degrade gracefully and let the healer work from stderr. + */ +async function tryReadSnapshot(sandbox: Sandbox): Promise<string | undefined> { + try { + const content = await sandbox.files.read(SNAPSHOT_PATH); + return typeof content === 'string' && content.length > 0 ? content : undefined; + } catch { + return undefined; + } +} + +export interface RunOptions { + /** Optional progress callback fired on each in-sandbox phase boundary. */ + onProgress?: (phase: RunPhase, detail?: string) => void; + /** Optional callback for raw stdout chunks as the sandbox emits them. */ + onStdoutChunk?: (chunk: string) => void; + /** Optional callback for raw stderr chunks as the sandbox emits them. */ + onStderrChunk?: (chunk: string) => void; +} + +/** + * Parse phase markers out of streamed output. + * + * The in-sandbox runner echoes `QMAX_PHASE:<phase>` lines at known + * boundaries; we surface those to the optional `onProgress` callback so + * UI consumers can show "Installing @playwright/test…", "Running test…" + * etc. without parsing arbitrary npm/Playwright output. The convention + * is borrowed from qualitymax.io's progress bar — a sentinel grep + * survives version bumps of npm/playwright that would break a regex on + * their normal log output. + */ +const PHASE_RE = /QMAX_PHASE:([a-z_]+)/g; + +function parsePhases( + chunk: string, + onProgress?: (phase: RunPhase, detail?: string) => void, +): void { + if (!onProgress) return; + let match: RegExpExecArray | null; + PHASE_RE.lastIndex = 0; + while ((match = PHASE_RE.exec(chunk)) !== null) { + onProgress(match[1] as RunPhase); + } +} + +/** + * Run a generated test once in a fresh sandbox. + * + * Caller is responsible for retry / healing -- this function returns + * structured output for one attempt only. + */ +export async function runInSandbox( + generated: GeneratedTest, + opts: RunOptions = {}, +): Promise<RunResult> { + opts.onProgress?.('sandbox_starting'); + const sandbox = await Sandbox.create(TEMPLATE); + try { + await prepareSandbox(sandbox, generated.code); + opts.onProgress?.('project_uploaded'); + await ensureTestRunner(sandbox); + opts.onProgress?.('deps_installed'); + + // E2B's SDK throws CommandExitError on non-zero exit rather than + // returning a result. For a test runner that's the wrong default -- + // a failed test IS the signal we want to capture and feed into the + // healing loop. Catch the typed error and use it directly, since + // CommandExitError implements CommandResult (exitCode/stdout/stderr + // are getters on the instance itself). + // + // PLAYWRIGHT_BROWSERS_PATH points at the template's pre-installed + // Chromium so @playwright/test doesn't try to download its own. + // + // The `echo QMAX_PHASE:test_started` markers are picked up by + // `parsePhases` and forwarded to onProgress -- see the comment on + // PHASE_RE for why we use sentinels instead of parsing npm output. + const cmd = + `echo QMAX_PHASE:test_started && ` + + `PLAYWRIGHT_BROWSERS_PATH=${BROWSERS_PATH} ` + + `npx playwright test --reporter=list 2>&1; ` + + `status=$?; echo QMAX_PHASE:test_finished; exit $status`; + + let exec: { exitCode: number; stdout: string; stderr: string }; + try { + exec = await sandbox.commands.run(cmd, { + cwd: WORK_DIR, + timeoutMs: 5 * 60 * 1000, + onStdout: (chunk: string) => { + opts.onStdoutChunk?.(chunk); + parsePhases(chunk, opts.onProgress); + }, + onStderr: (chunk: string) => { + opts.onStderrChunk?.(chunk); + parsePhases(chunk, opts.onProgress); + }, + }); + } catch (err) { + if (!(err instanceof CommandExitError)) throw err; + exec = err; + } + + const passed = exec.exitCode === 0; + const failureSnapshot = passed ? undefined : await tryReadSnapshot(sandbox); + opts.onProgress?.('artifacts_collected'); + + return { + passed, + exitCode: exec.exitCode, + stdout: exec.stdout, + stderr: exec.stderr, + failureSnapshot, + failureType: passed ? undefined : classifyFailure(exec.stdout, exec.stderr), + }; + } finally { + await sandbox.kill(); + } +} + +/** + * Classify a Playwright failure into one of a small set of buckets. + * + * The buckets are tuned for what the healer can actually act on: + * - `strict_mode_violation` → tell the LLM how to disambiguate + * (anchor by unique text, scope by parent role, use data-testid). + * - `locator_not_found` → broaden the selector strategy. + * - `timeout` → wait for network/load state, not bare element. + * - `assertion_failed` → likely the expected value changed, not the + * locator; tell the LLM to use a less specific assertion. + * + * Battle-tested ordering: strict-mode check before "locator …" because + * strict-mode errors *contain* "locator" in their message and would + * otherwise be mis-bucketed. + */ +export function classifyFailure( + stdout: string, + stderr: string, +): RunResult['failureType'] { + const text = `${stdout}\n${stderr}`.toLowerCase(); + if (/strict mode violation.*resolved to \d+ elements?/.test(text)) { + return 'strict_mode_violation'; + } + if (/locator|selector|waiting for/.test(text)) { + return 'locator_not_found'; + } + if (/timeout/.test(text)) return 'timeout'; + if (/assertion|expect\(/.test(text)) return 'assertion_failed'; + return 'unknown'; +} diff --git a/examples/self-healing-playwright-tests/src/types.ts b/examples/self-healing-playwright-tests/src/types.ts new file mode 100644 index 00000000..6d06c6ac --- /dev/null +++ b/examples/self-healing-playwright-tests/src/types.ts @@ -0,0 +1,98 @@ +/** + * Shared types for the self-healing test runner. + * + * The flow: + * 1. caller describes a test in natural language → TestSpec + * 2. router asks an LLM to write Playwright code → GeneratedTest + * 3. runner executes the test inside an E2B sandbox → RunResult + * 4. on failure, healer asks the LLM to fix it → HealAttempt + * 5. loop until pass or HEAL_MAX_ATTEMPTS is hit → HealingTrace + */ + +export type ProviderName = 'anthropic' | 'openai' | 'google'; + +/** + * Coarse classification of the failure so the healer can steer the next + * prompt. Battle-tested at qualitymax.io: without this, the healer treats + * "selector matches 4 elements" and "selector matches 0 elements" the + * same way and keeps emitting equivalent locators. + */ +export type FailureType = + | 'strict_mode_violation' + | 'locator_not_found' + | 'timeout' + | 'assertion_failed' + | 'unknown'; + +/** + * Sentinel phase emitted by the in-sandbox runner script. Surfaced via + * the optional `onProgress` callback on `runInSandbox` so callers driving + * a UI / CI can show real progress instead of an opaque "running…". + */ +export type RunPhase = + | 'sandbox_starting' + | 'project_uploaded' + | 'deps_installed' + | 'test_started' + | 'test_finished' + | 'artifacts_collected'; + +export interface TestSpec { + /** Natural-language description of what the test should do. */ + description: string; + /** Target URL the test should drive. */ + url: string; + /** Optional extra context for the LLM (selectors, expected text, etc.). */ + context?: string; +} + +export interface GeneratedTest { + /** TypeScript Playwright test file content. */ + code: string; + /** Provider that produced the code. */ + provider: ProviderName; + /** Token usage if reported by the provider. */ + usage?: { promptTokens?: number; completionTokens?: number }; +} + +export interface RunResult { + passed: boolean; + exitCode: number; + stdout: string; + stderr: string; + /** Snapshot of the failing page (HTML), if the runner managed to capture one. */ + failureSnapshot?: string; + /** Coarse classification of the failure, populated when !passed. */ + failureType?: FailureType; +} + +export interface HealAttempt { + attempt: number; + generated: GeneratedTest; + result: RunResult; +} + +export interface HealingTrace { + spec: TestSpec; + attempts: HealAttempt[]; + finalPassed: boolean; +} + +export interface RouterConfig { + /** Order in which providers are tried. */ + order: ProviderName[]; + /** Per-provider model id override. */ + models?: Partial<Record<ProviderName, string>>; +} + +export class AllProvidersFailedError extends Error { + constructor( + public attempts: { provider: ProviderName; error: Error }[], + ) { + const summary = attempts + .map((a) => `${a.provider}: ${a.error.message}`) + .join(' | '); + super(`All configured providers failed -- ${summary}`); + this.name = 'AllProvidersFailedError'; + } +} diff --git a/examples/self-healing-playwright-tests/tsconfig.json b/examples/self-healing-playwright-tests/tsconfig.json new file mode 100644 index 00000000..732ebadb --- /dev/null +++ b/examples/self-healing-playwright-tests/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "allowSyntheticDefaultImports": true + }, + "include": ["src/**/*", "examples/**/*"], + "exclude": ["node_modules", "dist"] +}