From 84d39d7b876edb198eea1bbeee3c611ea570abbd Mon Sep 17 00:00:00 2001 From: Ruslan Strazhnyk Date: Fri, 24 Apr 2026 21:31:42 +0200 Subject: [PATCH 1/2] feat(example): self-healing Playwright tests with multi-model fallback Adds a new cookbook example that demonstrates: - Generating a Playwright test from a natural-language spec via the Vercel AI SDK, with multi-model fallback (Claude -> GPT -> Gemini) so a single-provider outage doesn't stall the loop. - Running the generated test in the pre-baked `playwright-chromium` template (fresh sandbox per attempt). - On failure: capturing the page HTML via fs.writeFileSync, reading it back out of the sandbox, and feeding it into the next LLM call so the retry gets real DOM to anchor selectors on. Three small modules, ~250 lines total (router.ts / runner.ts / healer.ts). Four runnable examples: basic, self-healing, fallback, parallel suite. Pattern inspired by the self-healing behaviour at qualitymax.io; code is standalone TypeScript. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 1 + .../.env.template | 21 + .../self-healing-playwright-tests/README.md | 140 +++ .../examples/01-basic-test.ts | 53 + .../examples/02-self-healing.ts | 51 + .../examples/03-multi-model-fallback.ts | 52 + .../examples/04-parallel-suite.ts | 50 + .../package-lock.json | 991 ++++++++++++++++++ .../package.json | 46 + .../src/healer.ts | 130 +++ .../src/index.ts | 18 + .../src/router.ts | 135 +++ .../src/runner.ts | 132 +++ .../src/types.ts | 70 ++ .../tsconfig.json | 20 + 15 files changed, 1910 insertions(+) create mode 100644 examples/self-healing-playwright-tests/.env.template create mode 100644 examples/self-healing-playwright-tests/README.md create mode 100644 examples/self-healing-playwright-tests/examples/01-basic-test.ts create mode 100644 examples/self-healing-playwright-tests/examples/02-self-healing.ts create mode 100644 examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts create mode 100644 examples/self-healing-playwright-tests/examples/04-parallel-suite.ts create mode 100644 examples/self-healing-playwright-tests/package-lock.json create mode 100644 examples/self-healing-playwright-tests/package.json create mode 100644 examples/self-healing-playwright-tests/src/healer.ts create mode 100644 examples/self-healing-playwright-tests/src/index.ts create mode 100644 examples/self-healing-playwright-tests/src/router.ts create mode 100644 examples/self-healing-playwright-tests/src/runner.ts create mode 100644 examples/self-healing-playwright-tests/src/types.ts create mode 100644 examples/self-healing-playwright-tests/tsconfig.json diff --git a/README.md b/README.md index 5751550b..2bad893c 100644 --- a/README.md +++ b/README.md @@ -235,3 +235,4 @@ Read more about E2B on the [E2B website](https://e2b.dev) and the official [E2B - Next.js app with LLM + Code Interpreter and streaming - [TypeScript](./examples/nextjs-code-interpreter) - How to run a Docker container in E2B - [Python/TypeScript](./examples/docker-in-e2b) - How to run Playwright in E2B - [TypeScript](./examples/playwright-in-e2b) +- Self-healing Playwright tests with multi-model LLM fallback - [TypeScript](./examples/self-healing-playwright-tests) diff --git a/examples/self-healing-playwright-tests/.env.template b/examples/self-healing-playwright-tests/.env.template new file mode 100644 index 00000000..6f312bd4 --- /dev/null +++ b/examples/self-healing-playwright-tests/.env.template @@ -0,0 +1,21 @@ +# E2B sandbox API key — get one at https://e2b.dev +E2B_API_KEY= + +# At least one model provider key is required. +# The router will try them in the order they are configured (see ROUTER_ORDER). +ANTHROPIC_API_KEY= +OPENAI_API_KEY= +GOOGLE_GENERATIVE_AI_API_KEY= + +# Comma-separated provider order. Default: anthropic,openai,google +# Providers without a key are skipped automatically. +ROUTER_ORDER=anthropic,openai,google + +# How many self-healing attempts before giving up. Default: 3 +HEAL_MAX_ATTEMPTS=3 + +# Optional: pin specific models per provider. +# Defaults are sensible for early-2026 (Opus 4.7, GPT-5, Gemini 2.5 Pro). +ANTHROPIC_MODEL=claude-opus-4-7 +OPENAI_MODEL=gpt-5 +GOOGLE_MODEL=gemini-2.5-pro diff --git a/examples/self-healing-playwright-tests/README.md b/examples/self-healing-playwright-tests/README.md new file mode 100644 index 00000000..b5c50013 --- /dev/null +++ b/examples/self-healing-playwright-tests/README.md @@ -0,0 +1,140 @@ +# Self-Healing Playwright Tests with Multi-Model Fallback + +Generate a Playwright test from a natural-language spec, run it inside an isolated E2B sandbox, and — when it fails — feed the page snapshot back to the LLM so it rewrites the test with more robust selectors. Multi-model fallback (Claude → GPT → Gemini) via the Vercel AI SDK keeps the pipeline alive if one provider is down. + +The pattern is inspired by the self-healing behaviour used in production at [qualitymax.io](https://qualitymax.io); this is a standalone TypeScript example contributed to the E2B Cookbook. + +## Why + +Browser tests break the moment a button moves or a class name changes. Two common responses: + +1. Brittle CSS selectors + a human on call — the status quo. +2. Self-healing tests — capture the page state at failure, hand it to an LLM with the original spec, get back a fixed test. + +(2) only works if you can run the regenerated test in a clean, isolated environment without contaminating CI. E2B sandboxes give you exactly that: throwaway Firecracker microVMs, ~200 ms cold-start on the pre-baked `playwright-chromium` template. + +## Architecture + +``` + TestSpec (NL description + URL) + │ + ▼ + ┌──────────────┐ fallback chain (Claude → GPT → Gemini) + │ router.ts │ via Vercel AI SDK + └──────┬───────┘ + │ generated Playwright TS + ▼ + ┌──────────────┐ fresh sandbox per attempt + │ runner.ts │ runs test in `playwright-chromium` template + └──────┬───────┘ + │ pass / fail + stdout / stderr + page snapshot + ▼ + ┌──────────────┐ on fail: rewrite prompt with snapshot + errors + │ healer.ts │ back through router → runner, up to N attempts + └──────────────┘ +``` + +Three small modules, ~250 lines total: + +- **`src/router.ts`** — multi-model router. Reads `ROUTER_ORDER` from env, drops providers without keys, falls back on any error. +- **`src/runner.ts`** — E2B execution. Spins up `Sandbox.create('playwright-chromium')`, writes the generated test, installs `@playwright/test` in a user-writable work dir, executes, captures result. +- **`src/healer.ts`** — failure → snapshot → prompt → retry loop. + +## Setup + +```bash +npm install +cp .env.template .env # then fill in keys +npm run typecheck # sanity check +npm run example:basic # run the simplest example +``` + +You need: + +- An [E2B](https://e2b.dev) API key (free tier is enough). +- At least one of: `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GOOGLE_GENERATIVE_AI_API_KEY`. The router silently drops providers without keys, so a partial setup still works. + +Per attempt wall clock: **~30–80 s** (mostly LLM latency + test run). Full healing demo with 2–3 attempts: **~2–4 min**. For CI where you want the ~5–10 s `@playwright/test` install cost gone too, build your own E2B template on top of `playwright-chromium` with it pre-baked. + +## Examples + +| Script | What it shows | +| --- | --- | +| `examples/01-basic-test.ts` | Simplest path: spec → generated test → one sandboxed run, no healing. | +| `examples/02-self-healing.ts` | The headline. A vague spec produces a brittle test, the failure snapshot is fed back to the LLM, the retry passes. | +| `examples/03-multi-model-fallback.ts` | Simulate a provider outage by poisoning the Anthropic key. The router falls back to OpenAI / Google. Needs at least two provider keys configured to demonstrate anything meaningful. | +| `examples/04-parallel-suite.ts` | Run multiple healing tests concurrently in isolated sandboxes. | + +Run any of them with `npm run example:` (see `package.json`). + +## How healing actually works + +The non-obvious bit is what we feed back to the LLM after a failure. From `src/healer.ts`: + +1. **System rules** tell the LLM to dump `await page.content()` to `/app/failure.html` before throwing on any test failure. Without this, healing degrades to guessing from `stderr` alone. +2. **The runner** reads that snapshot off the sandbox filesystem after the failed run. +3. **The heal prompt** includes the previous code, the truncated stdout/stderr, and the page snapshot, with an explicit instruction to prefer role / accessible-name / data-testid selectors over brittle CSS paths. +4. **Each retry** spins up a fresh sandbox, so failures don't compound. + +`HEAL_MAX_ATTEMPTS` caps the loop (default `3`). + +### What a real run looks like + +``` +$ npm run example:healing + +→ attempt 1: LLM writes getByRole('link', { name: /Docs|Documentation/i }) + ✘ strict mode violation: locator resolved to 4 elements + → healer reads /app/failure.html, feeds DOM back to the LLM +→ attempt 2: LLM rewrites with a more specific selector + ✓ test passes + +Final: ✓ passed +Attempts: 2 + attempt 1 -- fail (generated by google) + attempt 2 -- pass (generated by google) +``` + +Each attempt spins up a fresh sandbox — failures from attempt 1 can't contaminate attempt 2. + +## Why multi-model fallback matters + +LLM provider availability is not 100%. On 15 April 2026 an Anthropic incident took down Claude API access for several hours. Single-provider agents went dark; anything with a router survived. + +`examples/03-multi-model-fallback.ts` reproduces the failure mode locally by poisoning the Anthropic key. You need at least two provider keys configured for this to show anything meaningful — with only one key the router has nowhere to fall back to. Output on a two-key setup: + +``` +→ Anthropic key is poisoned. Router should fall back. +✓ test generated -- the router landed on openai +✓ test passed +``` + +Production code will want to: +- Distinguish transient (5xx, rate limit) from permanent (auth) errors and retry transients on the same provider before falling back. +- Track provider-level success rates and reorder dynamically. +- Cap total wall-clock so a slow fallback doesn't block the loop. + +The pattern here is the floor, not the ceiling. + +## Notes on the `playwright-chromium` template + +The template ships `playwright` + browser binaries pre-installed at `/app`, but not `@playwright/test` (the test runner). The runner handles this: + +- `/app` is root-owned, so test files and additional npm installs go to `/home/user/work` instead. +- `@playwright/test` is installed there, pinned to `1.51.1` to match the template's bundled Playwright version (browser/runner version skew causes "chrome-headless-shell-: cannot execute binary" failures). +- `PLAYWRIGHT_BROWSERS_PATH` points at `/app/node_modules/playwright-core/.local-browsers` so the test runner re-uses the template's Chromium instead of downloading its own. + +All three bits are documented inline in `src/runner.ts`. + +## Adapting it + +- **Use a different test framework.** Replace the `ensureTestRunner` step in `src/runner.ts` and the `SYSTEM_RULES` in `src/healer.ts`. Cypress, Vitest browser mode, Pytest + Selenium — anything that runs in a Linux sandbox works. +- **Add more providers.** `src/router.ts` maps provider names to AI SDK clients. Add a case to `modelFor()` and an entry to `ENV_KEY_BY_PROVIDER`. +- **Build a custom E2B template.** Pre-bake `@playwright/test` on top of `playwright-chromium` to drop each attempt back toward the 200 ms cold-start floor. +- **Wire it into CI.** The whole entry point is `runHealingTest(spec, maxAttempts)`. Call it from a GitHub Action and post the trace to your build summary. + +## Credits + +- Self-healing pattern inspired by [qualitymax.io](https://qualitymax.io). +- Sandbox runtime: [E2B](https://e2b.dev). +- Model orchestration: [Vercel AI SDK](https://sdk.vercel.ai). diff --git a/examples/self-healing-playwright-tests/examples/01-basic-test.ts b/examples/self-healing-playwright-tests/examples/01-basic-test.ts new file mode 100644 index 00000000..2df11b71 --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/01-basic-test.ts @@ -0,0 +1,53 @@ +/** + * Example 1 -- the simplest path through the system. + * + * Generate a Playwright test from a one-line natural-language spec + * and run it once in a fresh E2B sandbox. No healing. + * + * Run: pnpm tsx examples/01-basic-test.ts + */ + +import 'dotenv/config'; + +import { route } from '../src/router.js'; +import { runInSandbox } from '../src/runner.js'; +import type { GeneratedTest, TestSpec } from '../src/types.js'; + +const spec: TestSpec = { + url: 'https://e2b.dev', + description: + 'The page should mention "AI agents" somewhere on it within 10 seconds of loading.', +}; + +const PROMPT = `You are an expert Playwright test author. +Output a single TypeScript file -- no markdown fences, no commentary. +Use @playwright/test, import { test, expect }. + +Spec: +URL: ${spec.url} +What to verify: ${spec.description} + +Return only the TypeScript file content.`; + +async function main(): Promise { + console.log('→ generating test...'); + const { text, provider } = await route(PROMPT); + const generated: GeneratedTest = { code: text.trim(), provider }; + console.log(`✓ generated by ${provider} (${generated.code.length} chars)`); + + console.log('→ running in E2B playwright-chromium sandbox (~30-50s)...'); + const result = await runInSandbox(generated); + console.log(result.passed ? '✓ test passed' : '✗ test failed'); + if (!result.passed) { + console.log('--- stdout ---'); + console.log(result.stdout); + console.log('--- stderr ---'); + console.log(result.stderr); + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/examples/02-self-healing.ts b/examples/self-healing-playwright-tests/examples/02-self-healing.ts new file mode 100644 index 00000000..85b2850e --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/02-self-healing.ts @@ -0,0 +1,51 @@ +/** + * Example 2 -- the headline example. + * + * Run a test that is *deliberately likely to fail on the first try* + * (description is vague enough that the LLM may pick brittle selectors). + * The healing loop kicks in: failure stdout + a captured page snapshot + * are fed back into the LLM, which rewrites the test with more robust + * selectors. We retry up to HEAL_MAX_ATTEMPTS times. + * + * This is the qualitymax.io self-healing pattern, ported to E2B. + * + * Run: pnpm tsx examples/02-self-healing.ts + */ + +import 'dotenv/config'; + +import { runHealingTest } from '../src/healer.js'; +import type { TestSpec } from '../src/types.js'; + +const spec: TestSpec = { + url: 'https://e2b.dev', + description: + 'Click whatever link or button appears to lead to the documentation, ' + + 'then verify the destination page contains the word "sandbox".', + context: + 'The site nav may have changed since the LLM was trained. Prefer ' + + 'role / accessible-name / data-testid based selectors.', +}; + +async function main(): Promise { + const trace = await runHealingTest(spec, 3); + + console.log(`\nFinal: ${trace.finalPassed ? '✓ passed' : '✗ failed'}`); + console.log(`Attempts: ${trace.attempts.length}`); + + trace.attempts.forEach((a) => { + console.log( + ` attempt ${a.attempt} -- ${a.result.passed ? 'pass' : 'fail'} ` + + `(generated by ${a.generated.provider})`, + ); + }); + + if (!trace.finalPassed) { + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts b/examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts new file mode 100644 index 00000000..640bb982 --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/03-multi-model-fallback.ts @@ -0,0 +1,52 @@ +/** + * Example 3 -- show the router fall back from a "broken" provider. + * + * We simulate the Anthropic outage of 15 April 2026 by overriding the + * Anthropic key with a guaranteed-bad value. The router should detect + * the auth failure and move on to OpenAI (or Google). The test still + * gets generated and run. + * + * This is the production behavior that kept qualitymax.io serving + * customers during that incident -- the architecture is the same here. + * + * Run: pnpm tsx examples/03-multi-model-fallback.ts + */ + +import 'dotenv/config'; + +import { route } from '../src/router.js'; +import { runInSandbox } from '../src/runner.js'; +import type { GeneratedTest } from '../src/types.js'; + +// Simulate the outage by poisoning Anthropic auth before the SDK reads it. +// We keep OpenAI / Google keys real so the fallback chain has somewhere +// to go. +process.env.ANTHROPIC_API_KEY = 'sk-ant-deliberately-invalid-for-demo'; + +const PROMPT = `You are an expert Playwright test author. +Output a single TypeScript file -- no markdown fences, no commentary. +Use @playwright/test, import { test, expect }. + +Spec: +URL: https://e2b.dev +What to verify: page is non-empty. + +Return only the TypeScript file content.`; + +async function main(): Promise<void> { + console.log('→ Anthropic key is poisoned. Router should fall back.'); + const { text, provider } = await route(PROMPT); + console.log(`✓ test generated -- the router landed on ${provider}`); + + const generated: GeneratedTest = { code: text.trim(), provider }; + const result = await runInSandbox(generated); + console.log(result.passed ? '✓ test passed' : '✗ test failed'); + if (!result.passed) { + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/examples/04-parallel-suite.ts b/examples/self-healing-playwright-tests/examples/04-parallel-suite.ts new file mode 100644 index 00000000..092fe7ad --- /dev/null +++ b/examples/self-healing-playwright-tests/examples/04-parallel-suite.ts @@ -0,0 +1,50 @@ +/** + * Example 4 -- run multiple healing tests in parallel sandboxes. + * + * Each TestSpec gets its own isolated E2B sandbox so failures and + * state don't leak between them. This is roughly how a real test + * suite would be wired up against E2B in production. + * + * Run: pnpm tsx examples/04-parallel-suite.ts + */ + +import 'dotenv/config'; + +import { runHealingTest } from '../src/healer.js'; +import type { TestSpec } from '../src/types.js'; + +const suite: TestSpec[] = [ + { + url: 'https://e2b.dev', + description: 'The page should mention "AI agents" somewhere on it.', + }, + { + url: 'https://e2b.dev/docs', + description: 'There should be a link to the Cookbook somewhere on the page.', + }, + { + url: 'https://e2b.dev/blog', + description: 'The page should list at least one blog post title.', + }, +]; + +async function main(): Promise<void> { + const traces = await Promise.all(suite.map((spec) => runHealingTest(spec, 2))); + + let failed = 0; + traces.forEach((trace) => { + const status = trace.finalPassed ? '✓' : '✗'; + console.log(`${status} ${trace.spec.url} (${trace.attempts.length} attempts)`); + if (!trace.finalPassed) failed++; + }); + + console.log(`\n${suite.length - failed}/${suite.length} passed`); + if (failed > 0) { + process.exitCode = 1; + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/examples/self-healing-playwright-tests/package-lock.json b/examples/self-healing-playwright-tests/package-lock.json new file mode 100644 index 00000000..c849badb --- /dev/null +++ b/examples/self-healing-playwright-tests/package-lock.json @@ -0,0 +1,991 @@ +{ + "name": "self-healing-playwright-tests", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "self-healing-playwright-tests", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/anthropic": "^1.2.0", + "@ai-sdk/google": "^1.2.0", + "@ai-sdk/openai": "^1.2.0", + "@e2b/code-interpreter": "^1.5.0", + "ai": "^4.3.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@ai-sdk/anthropic": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/anthropic/-/anthropic-1.2.12.tgz", + "integrity": "sha512-YSzjlko7JvuiyQFmI9RN1tNZdEiZxc+6xld/0tq/VkJaHpEzGAb1yiNxxvmYVcjvfu/PcvCxAAYXmTYQQ63IHQ==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/google": { + "version": "1.2.22", + "resolved": "https://registry.npmjs.org/@ai-sdk/google/-/google-1.2.22.tgz", + "integrity": "sha512-Ppxu3DIieF1G9pyQ5O1Z646GYR0gkC57YdBqXJ82qvCdhEhZHu0TWhmnOoeIWe2olSbuDeoOY+MfJrW8dzS3Hw==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/openai": { + "version": "1.3.24", + "resolved": "https://registry.npmjs.org/@ai-sdk/openai/-/openai-1.3.24.tgz", + "integrity": "sha512-GYXnGJTHRTZc4gJMSmFRgEQudjqd4PUN0ZjQhPwOAYH1yOAvQoG/Ikqs+HyISRbLPCrhbZnPKCNHuRU4OfpW0Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.0.0" + } + }, + "node_modules/@ai-sdk/provider": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider/-/provider-1.1.3.tgz", + "integrity": "sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==", + "license": "Apache-2.0", + "dependencies": { + "json-schema": "^0.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@ai-sdk/provider-utils": { + "version": "2.2.8", + "resolved": "https://registry.npmjs.org/@ai-sdk/provider-utils/-/provider-utils-2.2.8.tgz", + "integrity": "sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "nanoid": "^3.3.8", + "secure-json-parse": "^2.7.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@ai-sdk/react": { + "version": "1.2.12", + "resolved": "https://registry.npmjs.org/@ai-sdk/react/-/react-1.2.12.tgz", + "integrity": "sha512-jK1IZZ22evPZoQW3vlkZ7wvjYGYF+tRBKXtrcolduIkQ/m/sOAVcVeVDUDvh1T91xCnWCdUGCPZg2avZ90mv3g==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/ui-utils": "1.2.11", + "swr": "^2.2.5", + "throttleit": "2.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@ai-sdk/ui-utils": { + "version": "1.2.11", + "resolved": "https://registry.npmjs.org/@ai-sdk/ui-utils/-/ui-utils-1.2.11.tgz", + "integrity": "sha512-3zcwCc8ezzFlwp3ZD15wAPjf2Au4s3vAbKsXQVyhxODHcmu0iyPO2Eua6D/vicq/AUm/BAo60r97O6HU+EI0+w==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "zod-to-json-schema": "^3.24.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "zod": "^3.23.8" + } + }, + "node_modules/@bufbuild/protobuf": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-2.11.0.tgz", + "integrity": "sha512-sBXGT13cpmPR5BMgHE6UEEfEaShh5Ror6rfN3yEK5si7QVrtZg8LEPQb0VVhiLRUslD2yLnXtnRzG035J/mZXQ==", + "license": "(Apache-2.0 AND BSD-3-Clause)" + }, + "node_modules/@connectrpc/connect": { + "version": "2.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@connectrpc/connect/-/connect-2.0.0-rc.3.tgz", + "integrity": "sha512-ARBt64yEyKbanyRETTjcjJuHr2YXorzQo0etyS5+P6oSeW8xEuzajA9g+zDnMcj1hlX2dQE93foIWQGfpru7gQ==", + "license": "Apache-2.0", + "peerDependencies": { + "@bufbuild/protobuf": "^2.2.0" + } + }, + "node_modules/@connectrpc/connect-web": { + "version": "2.0.0-rc.3", + "resolved": "https://registry.npmjs.org/@connectrpc/connect-web/-/connect-web-2.0.0-rc.3.tgz", + "integrity": "sha512-w88P8Lsn5CCsA7MFRl2e6oLY4J/5toiNtJns/YJrlyQaWOy3RO8pDgkz+iIkG98RPMhj2thuBvsd3Cn4DKKCkw==", + "license": "Apache-2.0", + "peerDependencies": { + "@bufbuild/protobuf": "^2.2.0", + "@connectrpc/connect": "2.0.0-rc.3" + } + }, + "node_modules/@e2b/code-interpreter": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@e2b/code-interpreter/-/code-interpreter-1.5.1.tgz", + "integrity": "sha512-mkyKjAW2KN5Yt0R1I+1lbH3lo+W/g/1+C2lnwlitXk5wqi/g94SEO41XKdmDf5WWpKG3mnxWDR5d6S/lyjmMEw==", + "license": "MIT", + "dependencies": { + "e2b": "^1.4.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.7.tgz", + "integrity": "sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.7.tgz", + "integrity": "sha512-jbPXvB4Yj2yBV7HUfE2KHe4GJX51QplCN1pGbYjvsyCZbQmies29EoJbkEc+vYuU5o45AfQn37vZlyXy4YJ8RQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.7.tgz", + "integrity": "sha512-62dPZHpIXzvChfvfLJow3q5dDtiNMkwiRzPylSCfriLvZeq0a1bWChrGx/BbUbPwOrsWKMn8idSllklzBy+dgQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.7.tgz", + "integrity": "sha512-x5VpMODneVDb70PYV2VQOmIUUiBtY3D3mPBG8NxVk5CogneYhkR7MmM3yR/uMdITLrC1ml/NV1rj4bMJuy9MCg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.7.tgz", + "integrity": "sha512-5lckdqeuBPlKUwvoCXIgI2D9/ABmPq3Rdp7IfL70393YgaASt7tbju3Ac+ePVi3KDH6N2RqePfHnXkaDtY9fkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.7.tgz", + "integrity": "sha512-rYnXrKcXuT7Z+WL5K980jVFdvVKhCHhUwid+dDYQpH+qu+TefcomiMAJpIiC2EM3Rjtq0sO3StMV/+3w3MyyqQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.7.tgz", + "integrity": "sha512-B48PqeCsEgOtzME2GbNM2roU29AMTuOIN91dsMO30t+Ydis3z/3Ngoj5hhnsOSSwNzS+6JppqWsuhTp6E82l2w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.7.tgz", + "integrity": "sha512-jOBDK5XEjA4m5IJK3bpAQF9/Lelu/Z9ZcdhTRLf4cajlB+8VEhFFRjWgfy3M1O4rO2GQ/b2dLwCUGpiF/eATNQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.7.tgz", + "integrity": "sha512-RkT/YXYBTSULo3+af8Ib0ykH8u2MBh57o7q/DAs3lTJlyVQkgQvlrPTnjIzzRPQyavxtPtfg0EopvDyIt0j1rA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.7.tgz", + "integrity": "sha512-RZPHBoxXuNnPQO9rvjh5jdkRmVizktkT7TCDkDmQ0W2SwHInKCAV95GRuvdSvA7w4VMwfCjUiPwDi0ZO6Nfe9A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.7.tgz", + "integrity": "sha512-GA48aKNkyQDbd3KtkplYWT102C5sn/EZTY4XROkxONgruHPU72l+gW+FfF8tf2cFjeHaRbWpOYa/uRBz/Xq1Pg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.7.tgz", + "integrity": "sha512-a4POruNM2oWsD4WKvBSEKGIiWQF8fZOAsycHOt6JBpZ+JN2n2JH9WAv56SOyu9X5IqAjqSIPTaJkqN8F7XOQ5Q==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.7.tgz", + "integrity": "sha512-KabT5I6StirGfIz0FMgl1I+R1H73Gp0ofL9A3nG3i/cYFJzKHhouBV5VWK1CSgKvVaG4q1RNpCTR2LuTVB3fIw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.7.tgz", + "integrity": "sha512-gRsL4x6wsGHGRqhtI+ifpN/vpOFTQtnbsupUF5R5YTAg+y/lKelYR1hXbnBdzDjGbMYjVJLJTd2OFmMewAgwlQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.7.tgz", + "integrity": "sha512-hL25LbxO1QOngGzu2U5xeXtxXcW+/GvMN3ejANqXkxZ/opySAZMrc+9LY/WyjAan41unrR3YrmtTsUpwT66InQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.7.tgz", + "integrity": "sha512-2k8go8Ycu1Kb46vEelhu1vqEP+UeRVj2zY1pSuPdgvbd5ykAw82Lrro28vXUrRmzEsUV0NzCf54yARIK8r0fdw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.7.tgz", + "integrity": "sha512-hzznmADPt+OmsYzw1EE33ccA+HPdIqiCRq7cQeL1Jlq2gb1+OyWBkMCrYGBJ+sxVzve2ZJEVeePbLM2iEIZSxA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.7.tgz", + "integrity": "sha512-b6pqtrQdigZBwZxAn1UpazEisvwaIDvdbMbmrly7cDTMFnw/+3lVxxCTGOrkPVnsYIosJJXAsILG9XcQS+Yu6w==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.7.tgz", + "integrity": "sha512-OfatkLojr6U+WN5EDYuoQhtM+1xco+/6FSzJJnuWiUw5eVcicbyK3dq5EeV/QHT1uy6GoDhGbFpprUiHUYggrw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.7.tgz", + "integrity": "sha512-AFuojMQTxAz75Fo8idVcqoQWEHIXFRbOc1TrVcFSgCZtQfSdc1RXgB3tjOn/krRHENUB4j00bfGjyl2mJrU37A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.7.tgz", + "integrity": "sha512-+A1NJmfM8WNDv5CLVQYJ5PshuRm/4cI6WMZRg1by1GwPIQPCTs1GLEUHwiiQGT5zDdyLiRM/l1G0Pv54gvtKIg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.7.tgz", + "integrity": "sha512-+KrvYb/C8zA9CU/g0sR6w2RBw7IGc5J2BPnc3dYc5VJxHCSF1yNMxTV5LQ7GuKteQXZtspjFbiuW5/dOj7H4Yw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.7.tgz", + "integrity": "sha512-ikktIhFBzQNt/QDyOL580ti9+5mL/YZeUPKU2ivGtGjdTYoqz6jObj6nOMfhASpS4GU4Q/Clh1QtxWAvcYKamA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.7.tgz", + "integrity": "sha512-7yRhbHvPqSpRUV7Q20VuDwbjW5kIMwTHpptuUzV+AA46kiPze5Z7qgt6CLCK3pWFrHeNfDd1VKgyP4O+ng17CA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.7.tgz", + "integrity": "sha512-SmwKXe6VHIyZYbBLJrhOoCJRB/Z1tckzmgTLfFYOfpMAx63BJEaL9ExI8x7v0oAO3Zh6D/Oi1gVxEYr5oUCFhw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.7.tgz", + "integrity": "sha512-56hiAJPhwQ1R4i+21FVF7V8kSD5zZTdHcVuRFMW0hn753vVfQN8xlx4uOPT4xoGH0Z/oVATuR82AiqSTDIpaHg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@opentelemetry/api": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@types/diff-match-patch": { + "version": "1.0.36", + "resolved": "https://registry.npmjs.org/@types/diff-match-patch/-/diff-match-patch-1.0.36.tgz", + "integrity": "sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "22.19.17", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.19.17.tgz", + "integrity": "sha512-wGdMcf+vPYM6jikpS/qhg6WiqSV/OhG+jeeHT/KlVqxYfD40iYJf9/AE1uQxVWFvU7MipKRkRv8NSHiCGgPr8Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/ai": { + "version": "4.3.19", + "resolved": "https://registry.npmjs.org/ai/-/ai-4.3.19.tgz", + "integrity": "sha512-dIE2bfNpqHN3r6IINp9znguYdhIOheKW2LDigAMrgt/upT3B8eBGPSCblENvaZGoq+hxaN9fSMzjWpbqloP+7Q==", + "license": "Apache-2.0", + "dependencies": { + "@ai-sdk/provider": "1.1.3", + "@ai-sdk/provider-utils": "2.2.8", + "@ai-sdk/react": "1.2.12", + "@ai-sdk/ui-utils": "1.2.11", + "@opentelemetry/api": "1.9.0", + "jsondiffpatch": "0.6.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "react": "^18 || ^19 || ^19.0.0-rc", + "zod": "^3.23.8" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + } + } + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/compare-versions": { + "version": "6.1.1", + "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-6.1.1.tgz", + "integrity": "sha512-4hm4VPpIecmlg59CHXnRDnqGplJFrbLG4aFEl5vl6cK1u76ws3LLvX7ikFnTDl5vo39sjWD6AaDPYodJp/NNHg==", + "license": "MIT" + }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/diff-match-patch": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/diff-match-patch/-/diff-match-patch-1.0.5.tgz", + "integrity": "sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==", + "license": "Apache-2.0" + }, + "node_modules/dotenv": { + "version": "16.6.1", + "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz", + "integrity": "sha512-uBq4egWHTcTt33a72vpSG0z3HnPuIl6NqYcTrKEg2azoEyl2hpW0zqlxysq2pK9HlDIHyHyakeYaYnSAwd8bow==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://dotenvx.com" + } + }, + "node_modules/e2b": { + "version": "1.13.2", + "resolved": "https://registry.npmjs.org/e2b/-/e2b-1.13.2.tgz", + "integrity": "sha512-m8acE/MzMAJo1A57DakR2X1Sl5Mt1tcQO2aJfygNaQHLXby/4xsjF0UeJUB70jF7xntiR41pAMbZEHnkzrT9tw==", + "license": "MIT", + "dependencies": { + "@bufbuild/protobuf": "^2.6.2", + "@connectrpc/connect": "2.0.0-rc.3", + "@connectrpc/connect-web": "2.0.0-rc.3", + "compare-versions": "^6.1.0", + "openapi-fetch": "^0.9.7", + "platform": "^1.3.6" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/esbuild": { + "version": "0.27.7", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.7.tgz", + "integrity": "sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.7", + "@esbuild/android-arm": "0.27.7", + "@esbuild/android-arm64": "0.27.7", + "@esbuild/android-x64": "0.27.7", + "@esbuild/darwin-arm64": "0.27.7", + "@esbuild/darwin-x64": "0.27.7", + "@esbuild/freebsd-arm64": "0.27.7", + "@esbuild/freebsd-x64": "0.27.7", + "@esbuild/linux-arm": "0.27.7", + "@esbuild/linux-arm64": "0.27.7", + "@esbuild/linux-ia32": "0.27.7", + "@esbuild/linux-loong64": "0.27.7", + "@esbuild/linux-mips64el": "0.27.7", + "@esbuild/linux-ppc64": "0.27.7", + "@esbuild/linux-riscv64": "0.27.7", + "@esbuild/linux-s390x": "0.27.7", + "@esbuild/linux-x64": "0.27.7", + "@esbuild/netbsd-arm64": "0.27.7", + "@esbuild/netbsd-x64": "0.27.7", + "@esbuild/openbsd-arm64": "0.27.7", + "@esbuild/openbsd-x64": "0.27.7", + "@esbuild/openharmony-arm64": "0.27.7", + "@esbuild/sunos-x64": "0.27.7", + "@esbuild/win32-arm64": "0.27.7", + "@esbuild/win32-ia32": "0.27.7", + "@esbuild/win32-x64": "0.27.7" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/get-tsconfig": { + "version": "4.14.0", + "resolved": "https://registry.npmjs.org/get-tsconfig/-/get-tsconfig-4.14.0.tgz", + "integrity": "sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==", + "dev": true, + "license": "MIT", + "dependencies": { + "resolve-pkg-maps": "^1.0.0" + }, + "funding": { + "url": "https://github.com/privatenumber/get-tsconfig?sponsor=1" + } + }, + "node_modules/json-schema": { + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", + "integrity": "sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==", + "license": "(AFL-2.1 OR BSD-3-Clause)" + }, + "node_modules/jsondiffpatch": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/jsondiffpatch/-/jsondiffpatch-0.6.0.tgz", + "integrity": "sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==", + "license": "MIT", + "dependencies": { + "@types/diff-match-patch": "^1.0.36", + "chalk": "^5.3.0", + "diff-match-patch": "^1.0.5" + }, + "bin": { + "jsondiffpatch": "bin/jsondiffpatch.js" + }, + "engines": { + "node": "^18.0.0 || >=20.0.0" + } + }, + "node_modules/nanoid": { + "version": "3.3.11", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.11.tgz", + "integrity": "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/openapi-fetch": { + "version": "0.9.8", + "resolved": "https://registry.npmjs.org/openapi-fetch/-/openapi-fetch-0.9.8.tgz", + "integrity": "sha512-zM6elH0EZStD/gSiNlcPrzXcVQ/pZo3BDvC6CDwRDUt1dDzxlshpmQnpD6cZaJ39THaSmwVCxxRrPKNM1hHrDg==", + "license": "MIT", + "dependencies": { + "openapi-typescript-helpers": "^0.0.8" + } + }, + "node_modules/openapi-typescript-helpers": { + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/openapi-typescript-helpers/-/openapi-typescript-helpers-0.0.8.tgz", + "integrity": "sha512-1eNjQtbfNi5Z/kFhagDIaIRj6qqDzhjNJKz8cmMW0CVdGwT6e1GLbAfgI0d28VTJa1A8jz82jm/4dG8qNoNS8g==", + "license": "MIT" + }, + "node_modules/platform": { + "version": "1.3.6", + "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.6.tgz", + "integrity": "sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==", + "license": "MIT" + }, + "node_modules/react": { + "version": "19.2.5", + "resolved": "https://registry.npmjs.org/react/-/react-19.2.5.tgz", + "integrity": "sha512-llUJLzz1zTUBrskt2pwZgLq59AemifIftw4aB7JxOqf1HY2FDaGDxgwpAPVzHU1kdWabH7FauP4i1oEeer2WCA==", + "license": "MIT", + "peer": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/resolve-pkg-maps": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/resolve-pkg-maps/-/resolve-pkg-maps-1.0.0.tgz", + "integrity": "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/privatenumber/resolve-pkg-maps?sponsor=1" + } + }, + "node_modules/secure-json-parse": { + "version": "2.7.0", + "resolved": "https://registry.npmjs.org/secure-json-parse/-/secure-json-parse-2.7.0.tgz", + "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", + "license": "BSD-3-Clause" + }, + "node_modules/swr": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/swr/-/swr-2.4.1.tgz", + "integrity": "sha512-2CC6CiKQtEwaEeNiqWTAw9PGykW8SR5zZX8MZk6TeAvEAnVS7Visz8WzphqgtQ8v2xz/4Q5K+j+SeMaKXeeQIA==", + "license": "MIT", + "dependencies": { + "dequal": "^2.0.3", + "use-sync-external-store": "^1.6.0" + }, + "peerDependencies": { + "react": "^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/throttleit": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-2.1.0.tgz", + "integrity": "sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/tsx": { + "version": "4.21.0", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.21.0.tgz", + "integrity": "sha512-5C1sg4USs1lfG0GFb2RLXsdpXqBSEhAaA/0kPL01wxzpMqLILNxIxIOKiILz+cdg/pLnOUxFYOR5yhHU666wbw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.27.0", + "get-tsconfig": "^4.7.5" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/use-sync-external-store": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.6.0.tgz", + "integrity": "sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==", + "license": "MIT", + "peerDependencies": { + "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" + } + }, + "node_modules/zod": { + "version": "3.25.76", + "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", + "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + }, + "node_modules/zod-to-json-schema": { + "version": "3.25.2", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.25.2.tgz", + "integrity": "sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==", + "license": "ISC", + "peerDependencies": { + "zod": "^3.25.28 || ^4" + } + } + } +} diff --git a/examples/self-healing-playwright-tests/package.json b/examples/self-healing-playwright-tests/package.json new file mode 100644 index 00000000..f1a6f18a --- /dev/null +++ b/examples/self-healing-playwright-tests/package.json @@ -0,0 +1,46 @@ +{ + "name": "self-healing-playwright-tests", + "version": "0.1.0", + "description": "Cookbook example: self-healing browser tests on E2B sandboxes with Vercel AI SDK and multi-model fallback (Claude / GPT / Gemini).", + "license": "Apache-2.0", + "author": "Ruslan Strazhnyk <strazhnyk@gmail.com>", + "type": "module", + "scripts": { + "example:basic": "tsx examples/01-basic-test.ts", + "example:healing": "tsx examples/02-self-healing.ts", + "example:fallback": "tsx examples/03-multi-model-fallback.ts", + "example:suite": "tsx examples/04-parallel-suite.ts", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@ai-sdk/anthropic": "^1.2.0", + "@ai-sdk/google": "^1.2.0", + "@ai-sdk/openai": "^1.2.0", + "@e2b/code-interpreter": "^1.5.0", + "ai": "^4.3.0", + "dotenv": "^16.4.5", + "zod": "^3.23.8" + }, + "devDependencies": { + "@types/node": "^22.0.0", + "tsx": "^4.19.0", + "typescript": "^5.6.0" + }, + "engines": { + "node": ">=20" + }, + "keywords": [ + "e2b", + "ai-sdk", + "vercel-ai-sdk", + "playwright", + "self-healing-tests", + "ai-agents", + "multi-model", + "claude", + "gpt", + "gemini", + "cookbook", + "sandbox" + ] +} diff --git a/examples/self-healing-playwright-tests/src/healer.ts b/examples/self-healing-playwright-tests/src/healer.ts new file mode 100644 index 00000000..0543506d --- /dev/null +++ b/examples/self-healing-playwright-tests/src/healer.ts @@ -0,0 +1,130 @@ +/** + * Healing loop: generate test → run → on failure, ask LLM to fix it + * using the actual error output and a captured page snapshot, retry. + * + * Self-healing is the qualitymax.io feature this example demonstrates + * on E2B's substrate. The pattern is small enough to copy into any + * agent codebase that needs to keep brittle external tests running + * across UI changes. + */ + +import { + type GeneratedTest, + type HealAttempt, + type HealingTrace, + type TestSpec, +} from './types.js'; +import { route } from './router.js'; +import { runInSandbox } from './runner.js'; + +const DEFAULT_MAX_ATTEMPTS = Number.parseInt( + process.env.HEAL_MAX_ATTEMPTS ?? '3', + 10, +); + +/** + * Instructions the LLM gets for both first-generation and healing. + * + * Two non-obvious choices: + * - We explicitly ask for `await page.content()` to be written to + * /home/user/failure.html on failure. The runner reads that snapshot + * and feeds it back into the next heal attempt -- without this, + * healing degrades to guessing from stderr alone. + * - We forbid markdown fences in the response so the runner can pipe + * the text straight into a .ts file. + */ +const SYSTEM_RULES = `You are an expert Playwright test author. +Output a single TypeScript file -- no markdown fences, no commentary. +The file must use @playwright/test and import { test, expect }. +On failure, before throwing, write the current page HTML to /app/failure.html +using fs.writeFileSync so a follow-up agent can inspect the page state.`; + +function buildGeneratePrompt(spec: TestSpec): string { + return `${SYSTEM_RULES} + +Write a Playwright test for the following spec: + +URL: ${spec.url} +What to verify: ${spec.description} +${spec.context ? `Extra context: ${spec.context}` : ''} + +Return only the TypeScript file content.`; +} + +function buildHealPrompt( + spec: TestSpec, + previous: GeneratedTest, + failure: { stdout: string; stderr: string; snapshot?: string }, +): string { + const snapshotBlock = failure.snapshot + ? `\nPage HTML at failure (truncated to 8000 chars):\n${failure.snapshot.slice(0, 8000)}\n` + : '\nNo page snapshot was captured.\n'; + + return `${SYSTEM_RULES} + +The previous test below failed. Fix it. + +Spec: +URL: ${spec.url} +What to verify: ${spec.description} +${spec.context ? `Extra context: ${spec.context}` : ''} + +Previous test: +${previous.code} + +Failure stdout: +${failure.stdout.slice(0, 4000)} + +Failure stderr: +${failure.stderr.slice(0, 2000)} +${snapshotBlock} +Return only the new TypeScript file content. Make selectors more robust; +prefer role / text / data-testid based selectors over brittle CSS paths.`; +} + +async function generate(spec: TestSpec): Promise<GeneratedTest> { + const { text, provider, usage } = await route(buildGeneratePrompt(spec)); + return { code: text.trim(), provider, usage }; +} + +async function heal( + spec: TestSpec, + previous: HealAttempt, +): Promise<GeneratedTest> { + const { text, provider, usage } = await route( + buildHealPrompt(spec, previous.generated, { + stdout: previous.result.stdout, + stderr: previous.result.stderr, + snapshot: previous.result.failureSnapshot, + }), + ); + return { code: text.trim(), provider, usage }; +} + +/** + * Drive a test from natural-language spec to passing run, healing on + * each failure. Returns the full attempt log so the caller can inspect + * which provider produced which version. + */ +export async function runHealingTest( + spec: TestSpec, + maxAttempts: number = DEFAULT_MAX_ATTEMPTS, +): Promise<HealingTrace> { + const attempts: HealAttempt[] = []; + let generated = await generate(spec); + + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const result = await runInSandbox(generated); + attempts.push({ attempt, generated, result }); + + if (result.passed) { + return { spec, attempts, finalPassed: true }; + } + if (attempt === maxAttempts) { + break; + } + generated = await heal(spec, attempts[attempts.length - 1]!); + } + + return { spec, attempts, finalPassed: false }; +} diff --git a/examples/self-healing-playwright-tests/src/index.ts b/examples/self-healing-playwright-tests/src/index.ts new file mode 100644 index 00000000..faf8421f --- /dev/null +++ b/examples/self-healing-playwright-tests/src/index.ts @@ -0,0 +1,18 @@ +/** + * Public entry points for the cookbook example. + */ + +export { runHealingTest } from './healer.js'; +export { route, loadRouterConfig } from './router.js'; +export { runInSandbox } from './runner.js'; + +export type { + TestSpec, + GeneratedTest, + RunResult, + HealAttempt, + HealingTrace, + RouterConfig, + ProviderName, +} from './types.js'; +export { AllProvidersFailedError } from './types.js'; diff --git a/examples/self-healing-playwright-tests/src/router.ts b/examples/self-healing-playwright-tests/src/router.ts new file mode 100644 index 00000000..cbcdbaec --- /dev/null +++ b/examples/self-healing-playwright-tests/src/router.ts @@ -0,0 +1,135 @@ +/** + * Multi-model router with fallback chain. + * + * Tries providers in order. On rate-limit, 5xx, or network error, + * falls back to the next configured provider. Throws AllProvidersFailedError + * if every provider fails for the same call. + * + * This is the same pattern qualitymax.io uses to survive single-provider + * outages (the platform stayed up through the documented Anthropic + * outage of 15 April 2026 because the router fell back to GPT and Gemini). + */ + +import { generateText } from 'ai'; +import type { LanguageModelV1 } from 'ai'; +import { anthropic } from '@ai-sdk/anthropic'; +import { openai } from '@ai-sdk/openai'; +import { google } from '@ai-sdk/google'; + +import { + AllProvidersFailedError, + type ProviderName, + type RouterConfig, +} from './types.js'; + +const DEFAULT_MODELS: Record<ProviderName, string> = { + anthropic: process.env.ANTHROPIC_MODEL ?? 'claude-opus-4-7', + openai: process.env.OPENAI_MODEL ?? 'gpt-5', + google: process.env.GOOGLE_MODEL ?? 'gemini-2.5-pro', +}; + +const ENV_KEY_BY_PROVIDER: Record<ProviderName, string> = { + anthropic: 'ANTHROPIC_API_KEY', + openai: 'OPENAI_API_KEY', + google: 'GOOGLE_GENERATIVE_AI_API_KEY', +}; + +function modelFor(provider: ProviderName, override?: string): LanguageModelV1 { + const id = override ?? DEFAULT_MODELS[provider]; + switch (provider) { + case 'anthropic': + return anthropic(id); + case 'openai': + return openai(id); + case 'google': + return google(id); + } +} + +/** + * Read the router config from env vars, with sensible defaults. + * + * Providers without a configured API key are silently dropped from the + * order so that a partial setup still works. + */ +export function loadRouterConfig(): RouterConfig { + const raw = process.env.ROUTER_ORDER ?? 'anthropic,openai,google'; + const requested = raw + .split(',') + .map((s) => s.trim().toLowerCase()) + .filter((s): s is ProviderName => + s === 'anthropic' || s === 'openai' || s === 'google', + ); + const order = requested.filter((p) => Boolean(process.env[ENV_KEY_BY_PROVIDER[p]])); + if (order.length === 0) { + throw new Error( + 'No usable provider keys found. Set at least one of ' + + `${Object.values(ENV_KEY_BY_PROVIDER).join(', ')} in your environment.`, + ); + } + return { order }; +} + +export interface RouteCallResult { + text: string; + provider: ProviderName; + usage?: { promptTokens?: number; completionTokens?: number }; +} + +/** + * Strip a single outer markdown fence if present. + * + * LLMs occasionally wrap their response in ```lang ... ``` despite + * prompts that explicitly forbid it (Gemini is the usual offender). + * Since every caller in this cookbook expects raw TypeScript that can + * be written straight to a .ts file, we normalise at the router. + * + * Surgical by design: only strips when the trimmed text starts AND + * ends with a fence, so response bodies that legitimately contain + * inline fences are left alone. + */ +function stripCodeFences(text: string): string { + const trimmed = text.trim(); + const match = trimmed.match(/^```[^\n]*\n([\s\S]*?)\n?```$/); + return match ? match[1]! : trimmed; +} + +/** + * Try the configured providers in order until one returns a response. + * + * The error classification here is intentionally conservative -- on any + * thrown error we move on to the next provider. Production code may want + * to retry transient errors on the same provider before falling back. + */ +export async function route( + prompt: string, + config: RouterConfig = loadRouterConfig(), +): Promise<RouteCallResult> { + const failures: { provider: ProviderName; error: Error }[] = []; + + for (const provider of config.order) { + try { + const model = modelFor(provider, config.models?.[provider]); + const { text, usage } = await generateText({ + model, + prompt, + // Slightly lower temperature to keep generated test code stable. + temperature: 0.2, + }); + return { + text: stripCodeFences(text), + provider, + usage: { + promptTokens: usage?.promptTokens, + completionTokens: usage?.completionTokens, + }, + }; + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + failures.push({ provider, error: e }); + // Loop continues to the next provider in `order`. + } + } + + throw new AllProvidersFailedError(failures); +} diff --git a/examples/self-healing-playwright-tests/src/runner.ts b/examples/self-healing-playwright-tests/src/runner.ts new file mode 100644 index 00000000..9d0f0a02 --- /dev/null +++ b/examples/self-healing-playwright-tests/src/runner.ts @@ -0,0 +1,132 @@ +/** + * E2B sandbox runner: writes a generated Playwright test file into a + * fresh sandbox on E2B's `playwright-chromium` template (Playwright + + * Chromium + system deps pre-baked), executes the test, and captures + * pass/fail plus stdout/stderr. + * + * Each call spins up a new sandbox so failures are isolated. Cold start + * is ~200ms on the pre-baked template; swap to `base` and add an + * `installPlaywright` step if you need to demonstrate bring-your-own. + */ + +import { CommandExitError, Sandbox } from '@e2b/code-interpreter'; + +import type { GeneratedTest, RunResult } from './types.js'; + +const TEMPLATE = 'playwright-chromium'; + +// Two-directory layout: the template's /app is owned by root and +// contains the pre-baked Playwright browser binaries, so we can't +// write our test files or install @playwright/test there. /home/user +// is user-writable; we keep tests and the test-runner package here and +// point PLAYWRIGHT_BROWSERS_PATH at /app's browser cache. +const WORK_DIR = '/home/user/work'; +const BROWSERS_PATH = '/app/node_modules/playwright-core/.local-browsers'; +const TEST_FILE_PATH = `${WORK_DIR}/test.spec.ts`; +const SNAPSHOT_PATH = `${WORK_DIR}/failure.html`; + +// Pin @playwright/test to match the version of `playwright` bundled in +// the template (check with `cat /app/package.json` in a probe sandbox). +// Version skew between the test runner and the browser binaries causes +// "chrome-headless-shell-<rev>: cannot execute binary" type failures. +const PLAYWRIGHT_VERSION = '1.51.1'; + +/** + * Write the generated test into the sandbox along with a small Playwright + * config that drops the test runner into the working directory. + */ +async function prepareSandbox(sandbox: Sandbox, code: string): Promise<void> { + await sandbox.files.write(TEST_FILE_PATH, code); + await sandbox.files.write( + `${WORK_DIR}/playwright.config.ts`, + `import { defineConfig } from '@playwright/test'; +export default defineConfig({ + testDir: '${WORK_DIR}', + reporter: 'list', + use: { headless: true }, +}); +`, + ); +} + +/** + * The `playwright-chromium` template ships with `playwright` + browser + * binaries pre-installed at /app, but not `@playwright/test` (the test + * runner). Create a user-writable work dir and install just that -- + * cheap (~5-10s) compared to a full browser install. + * + * If E2B later adds `@playwright/test` to the template at a location + * we can resolve from, this function can be deleted. + */ +async function ensureTestRunner(sandbox: Sandbox): Promise<void> { + await sandbox.commands.run( + `mkdir -p ${WORK_DIR} && ` + + `cd ${WORK_DIR} && ` + + `npm init -y >/dev/null && ` + + `npm install --no-audit --no-fund --silent @playwright/test@${PLAYWRIGHT_VERSION}`, + { timeoutMs: 120 * 1000 }, + ); +} + +/** + * Best-effort capture of the page state at failure time. Reads the + * snapshot file if the test wrote one (see HEALING_INSTRUCTIONS in + * healer.ts which asks the LLM to dump page.content() on failure). + */ +async function tryReadSnapshot(sandbox: Sandbox): Promise<string | undefined> { + try { + const content = await sandbox.files.read(SNAPSHOT_PATH); + return typeof content === 'string' ? content : undefined; + } catch { + return undefined; + } +} + +/** + * Run a generated test once in a fresh sandbox. + * + * Caller is responsible for retry / healing -- this function returns + * structured output for one attempt only. + */ +export async function runInSandbox( + generated: GeneratedTest, +): Promise<RunResult> { + const sandbox = await Sandbox.create(TEMPLATE); + try { + await prepareSandbox(sandbox, generated.code); + await ensureTestRunner(sandbox); + + // E2B's SDK throws CommandExitError on non-zero exit rather than + // returning a result. For a test runner that's the wrong default -- + // a failed test IS the signal we want to capture and feed into the + // healing loop. Catch the typed error and use it directly, since + // CommandExitError implements CommandResult (exitCode/stdout/stderr + // are getters on the instance itself). + // + // PLAYWRIGHT_BROWSERS_PATH points at the template's pre-installed + // Chromium so @playwright/test doesn't try to download its own. + let exec: { exitCode: number; stdout: string; stderr: string }; + try { + exec = await sandbox.commands.run( + `PLAYWRIGHT_BROWSERS_PATH=${BROWSERS_PATH} npx playwright test --reporter=list 2>&1`, + { cwd: WORK_DIR, timeoutMs: 5 * 60 * 1000 }, + ); + } catch (err) { + if (!(err instanceof CommandExitError)) throw err; + exec = err; + } + + const passed = exec.exitCode === 0; + const failureSnapshot = passed ? undefined : await tryReadSnapshot(sandbox); + + return { + passed, + exitCode: exec.exitCode, + stdout: exec.stdout, + stderr: exec.stderr, + failureSnapshot, + }; + } finally { + await sandbox.kill(); + } +} diff --git a/examples/self-healing-playwright-tests/src/types.ts b/examples/self-healing-playwright-tests/src/types.ts new file mode 100644 index 00000000..dab55de2 --- /dev/null +++ b/examples/self-healing-playwright-tests/src/types.ts @@ -0,0 +1,70 @@ +/** + * Shared types for the self-healing test runner. + * + * The flow: + * 1. caller describes a test in natural language → TestSpec + * 2. router asks an LLM to write Playwright code → GeneratedTest + * 3. runner executes the test inside an E2B sandbox → RunResult + * 4. on failure, healer asks the LLM to fix it → HealAttempt + * 5. loop until pass or HEAL_MAX_ATTEMPTS is hit → HealingTrace + */ + +export type ProviderName = 'anthropic' | 'openai' | 'google'; + +export interface TestSpec { + /** Natural-language description of what the test should do. */ + description: string; + /** Target URL the test should drive. */ + url: string; + /** Optional extra context for the LLM (selectors, expected text, etc.). */ + context?: string; +} + +export interface GeneratedTest { + /** TypeScript Playwright test file content. */ + code: string; + /** Provider that produced the code. */ + provider: ProviderName; + /** Token usage if reported by the provider. */ + usage?: { promptTokens?: number; completionTokens?: number }; +} + +export interface RunResult { + passed: boolean; + exitCode: number; + stdout: string; + stderr: string; + /** Snapshot of the failing page (HTML), if the runner managed to capture one. */ + failureSnapshot?: string; +} + +export interface HealAttempt { + attempt: number; + generated: GeneratedTest; + result: RunResult; +} + +export interface HealingTrace { + spec: TestSpec; + attempts: HealAttempt[]; + finalPassed: boolean; +} + +export interface RouterConfig { + /** Order in which providers are tried. */ + order: ProviderName[]; + /** Per-provider model id override. */ + models?: Partial<Record<ProviderName, string>>; +} + +export class AllProvidersFailedError extends Error { + constructor( + public attempts: { provider: ProviderName; error: Error }[], + ) { + const summary = attempts + .map((a) => `${a.provider}: ${a.error.message}`) + .join(' | '); + super(`All configured providers failed -- ${summary}`); + this.name = 'AllProvidersFailedError'; + } +} diff --git a/examples/self-healing-playwright-tests/tsconfig.json b/examples/self-healing-playwright-tests/tsconfig.json new file mode 100644 index 00000000..732ebadb --- /dev/null +++ b/examples/self-healing-playwright-tests/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "lib": ["ES2022"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true, + "noImplicitOverride": true, + "noFallthroughCasesInSwitch": true, + "allowSyntheticDefaultImports": true + }, + "include": ["src/**/*", "examples/**/*"], + "exclude": ["node_modules", "dist"] +} From 509c5122fb1ccc2de5b4da912979b7cda84001a5 Mon Sep 17 00:00:00 2001 From: Ruslan Strazhnyk <strazhnyk@gmail.com> Date: Wed, 13 May 2026 22:08:48 +0200 Subject: [PATCH 2/2] Add battle-tested patterns to self-healing example Three updates rolled in from production use at qualitymax.io: 1. Inject test.afterEach capture hook server-side with sentinel-bracketed strip-and-reattach instead of trusting the LLM to write the snapshot. Closes three silent failure modes -- LLM omits the hook on heal, writes to root-owned /app/ (EACCES), or duplicates `import fs` producing a SyntaxError that fails every retry before the test runs. Also fixes a path bug: the prompt told the LLM to write to /app/failure.html but the runner read /home/user/work/failure.html, so the snapshot was never actually loaded. 2. Classify failures (strict_mode_violation / locator_not_found / timeout / assertion_failed / unknown) and only inject a strict-mode hint on the matching bucket. Without targeted guidance, the healer re-emits equivalent fragile locators and burns the heal budget on the same failure. The strict-mode hint points at three concrete fix patterns instead. 3. Sentinel-based progress (QMAX_PHASE:* markers) parsed from streaming onStdout/onStderr so UI consumers see real-time progress instead of an opaque "running...". README updated with a "Battle-tested patterns" section explaining each. typecheck clean. --- .../self-healing-playwright-tests/README.md | 41 +++- .../src/healer.ts | 67 +++++-- .../src/index.ts | 11 +- .../src/runner.ts | 185 +++++++++++++++++- .../src/types.ts | 28 +++ 5 files changed, 302 insertions(+), 30 deletions(-) diff --git a/examples/self-healing-playwright-tests/README.md b/examples/self-healing-playwright-tests/README.md index b5c50013..22c67ebe 100644 --- a/examples/self-healing-playwright-tests/README.md +++ b/examples/self-healing-playwright-tests/README.md @@ -69,12 +69,13 @@ Run any of them with `npm run example:<name>` (see `package.json`). ## How healing actually works -The non-obvious bit is what we feed back to the LLM after a failure. From `src/healer.ts`: +The non-obvious bit is what we feed back to the LLM after a failure: -1. **System rules** tell the LLM to dump `await page.content()` to `/app/failure.html` before throwing on any test failure. Without this, healing degrades to guessing from `stderr` alone. +1. **The runner injects a `test.afterEach` capture hook** into the generated test before each attempt (sentinel-bracketed, stripped and reattached on every heal). On any failed test it writes `await page.content()` to a known path inside the sandbox. The LLM never has to remember to do this — see "Battle-tested patterns" below for why that matters. 2. **The runner** reads that snapshot off the sandbox filesystem after the failed run. -3. **The heal prompt** includes the previous code, the truncated stdout/stderr, and the page snapshot, with an explicit instruction to prefer role / accessible-name / data-testid selectors over brittle CSS paths. -4. **Each retry** spins up a fresh sandbox, so failures don't compound. +3. **The healer classifies the failure** (`strict_mode_violation`, `locator_not_found`, `timeout`, `assertion_failed`, `unknown`) and steers the prompt accordingly. Strict-mode violations in particular get a dedicated hint pointing at three concrete fix patterns instead of letting the LLM re-emit equivalent fragile locators. +4. **The heal prompt** includes the previous code, the truncated stdout/stderr, the page snapshot, and the failure-type-specific hint. +5. **Each retry** spins up a fresh sandbox, so failures don't compound. `HEAL_MAX_ATTEMPTS` caps the loop (default `3`). @@ -97,6 +98,38 @@ Attempts: 2 Each attempt spins up a fresh sandbox — failures from attempt 1 can't contaminate attempt 2. +## Battle-tested patterns + +The first cut of this example trusted the LLM to do more of the wiring. Three problems showed up in production at [qualitymax.io](https://qualitymax.io) and got rolled back into the code here. If you copy this pattern into your own agent, these are the ones worth keeping: + +### 1. Inject the page-snapshot hook yourself; don't ask the LLM to write it + +Original approach: tell the LLM to call `fs.writeFileSync(somePath, await page.content())` in its own `afterEach`. Three failure modes, none loud: + +- LLM omits the hook in the heal pass → no snapshot → healing degrades to guessing from `stderr`. +- LLM writes to a root-owned path (`/app/...`) → `EACCES` → silent, the test still fails for the original reason but you have no DOM. +- LLM duplicates `import fs from 'fs'` on heal → `SyntaxError`, every retry now fails before the test runs. + +Fix: `src/runner.ts` injects a controlled `test.afterEach` block between sentinels (`// --- begin self-healing capture hook ---` / `// --- end ---`) and strip-and-reattaches it on every heal. The hook uses `require('fs')` inline instead of a top-level import so a duplicated injection can't break the file. + +### 2. Classify the failure before prompting + +`src/runner.ts → classifyFailure()` buckets failures into one of five types. The healer only injects the strict-mode hint when `strict_mode_violation` matched. Without classification, the strict-mode guidance pollutes prompts for unrelated failures (timeouts, assertion mismatches) and pushes the LLM toward "fixing" non-issues. + +Strict-mode-violation handling is the highest-leverage instance: without the explicit hint, the LLM kept swapping `.first()` in or trying a different `[class*=…]` matcher and burning the whole heal budget on the same failure. With the hint pointing at "anchor by unique text and walk up" / "scope by parent role" / "use the exact `data-test` from the snapshot", it usually heals in one extra attempt. + +### 3. Sentinel-based progress, not log scraping + +`src/runner.ts` emits `QMAX_PHASE:<phase>` markers at known points in the in-sandbox command and forwards them via the optional `onProgress` callback. A grep for a sentinel survives the npm/Playwright version bump that would break a regex over their normal log output. The pattern reads from `commands.run({ onStdout, onStderr })` rather than waiting for the run to finish, so UI consumers see progress in real time. + +```ts +const result = await runInSandbox(generated, { + onProgress: (phase) => console.log(`[${phase}]`), +}); +``` + +If you want a *live browser feed* on top of progress markers (watch the test drive Chromium while it runs), see the [playwright-live-vnc-feed](../playwright-live-vnc-feed) example. + ## Why multi-model fallback matters LLM provider availability is not 100%. On 15 April 2026 an Anthropic incident took down Claude API access for several hours. Single-provider agents went dark; anything with a router survived. diff --git a/examples/self-healing-playwright-tests/src/healer.ts b/examples/self-healing-playwright-tests/src/healer.ts index 0543506d..3a5824bd 100644 --- a/examples/self-healing-playwright-tests/src/healer.ts +++ b/examples/self-healing-playwright-tests/src/healer.ts @@ -9,6 +9,7 @@ */ import { + type FailureType, type GeneratedTest, type HealAttempt, type HealingTrace, @@ -25,19 +26,24 @@ const DEFAULT_MAX_ATTEMPTS = Number.parseInt( /** * Instructions the LLM gets for both first-generation and healing. * - * Two non-obvious choices: - * - We explicitly ask for `await page.content()` to be written to - * /home/user/failure.html on failure. The runner reads that snapshot - * and feeds it back into the next heal attempt -- without this, - * healing degrades to guessing from stderr alone. - * - We forbid markdown fences in the response so the runner can pipe - * the text straight into a .ts file. + * Note what we *don't* ask for: the LLM no longer has to remember to + * dump page.content() on failure. The runner injects a controlled + * `test.afterEach` hook for that (see `injectCaptureHook` in runner.ts). + * Asking the LLM to do it produced three battle-tested failure modes + * at qualitymax.io — see the hook's inline comment for the full story. + * + * We also forbid markdown fences so the runner can pipe the text + * straight into a .ts file. Gemini in particular still occasionally + * wraps output in fences; the router has a defensive `stripCodeFences` + * pass for that, but the prompt belt-and-braces it. */ const SYSTEM_RULES = `You are an expert Playwright test author. Output a single TypeScript file -- no markdown fences, no commentary. The file must use @playwright/test and import { test, expect }. -On failure, before throwing, write the current page HTML to /app/failure.html -using fs.writeFileSync so a follow-up agent can inspect the page state.`; +Prefer robust selectors: getByRole / getByText / data-testid / data-test. +When multiple elements could match a locator, anchor by unique text and +walk up to a container, or scope by parent role. Never use [class*=...] +substring matchers -- they trigger strict-mode violations on real pages.`; function buildGeneratePrompt(spec: TestSpec): string { return `${SYSTEM_RULES} @@ -51,15 +57,51 @@ ${spec.context ? `Extra context: ${spec.context}` : ''} Return only the TypeScript file content.`; } +/** + * Hint appended to the heal prompt for strict-mode violations. + * + * Battle-tested at qualitymax.io: without this, the healer treats a + * "resolved to 4 elements" error the same as "resolved to 0 elements" + * and frequently re-emits an equivalent fragile locator (e.g. swaps + * `.first()` in, swaps another `[class*=…]` in). Burns the whole heal + * budget on the same failure. Steering the LLM to one of three concrete + * patterns turned the strict-mode failure mode from "blocks healing" + * into "heals in one extra attempt" in our internal metrics. + */ +const STRICT_MODE_HINT = ` +=== STRICT-MODE VIOLATION DETECTED === +The failing locator matches multiple elements. Playwright's strict mode +rejects ambiguous locators. Do NOT add \`.first()\` blindly -- almost always +the right fix is one of: + 1. Anchor by unique text and walk up to the container: + page.getByText('Total Balance').locator('..').getByText(/\\$\\d+/) + 2. Scope by parent role/region: + page.getByRole('region', { name: 'Total Balance' }).getByText(/\\$\\d+/) + 3. Use the exact data-test attribute if one is in the page snapshot: + page.locator('[data-test="total-balance"]') +Substring class matchers like [class*="balance"] are forbidden -- they always +hit multiple elements on a real page. Replace them with one of the above.`; + function buildHealPrompt( spec: TestSpec, previous: GeneratedTest, - failure: { stdout: string; stderr: string; snapshot?: string }, + failure: { + stdout: string; + stderr: string; + snapshot?: string; + failureType?: FailureType; + }, ): string { const snapshotBlock = failure.snapshot ? `\nPage HTML at failure (truncated to 8000 chars):\n${failure.snapshot.slice(0, 8000)}\n` : '\nNo page snapshot was captured.\n'; + // Only inject the strict-mode hint when the failure is actually a + // strict-mode violation. Including it unconditionally pollutes prompts + // for unrelated failures (assertion mismatches, timeouts) and pushes + // the LLM toward "fixing" non-issues. + const failureHint = failure.failureType === 'strict_mode_violation' ? STRICT_MODE_HINT : ''; + return `${SYSTEM_RULES} The previous test below failed. Fix it. @@ -77,9 +119,9 @@ ${failure.stdout.slice(0, 4000)} Failure stderr: ${failure.stderr.slice(0, 2000)} +${failureHint} ${snapshotBlock} -Return only the new TypeScript file content. Make selectors more robust; -prefer role / text / data-testid based selectors over brittle CSS paths.`; +Return only the new TypeScript file content.`; } async function generate(spec: TestSpec): Promise<GeneratedTest> { @@ -96,6 +138,7 @@ async function heal( stdout: previous.result.stdout, stderr: previous.result.stderr, snapshot: previous.result.failureSnapshot, + failureType: previous.result.failureType, }), ); return { code: text.trim(), provider, usage }; diff --git a/examples/self-healing-playwright-tests/src/index.ts b/examples/self-healing-playwright-tests/src/index.ts index faf8421f..9c07e8ab 100644 --- a/examples/self-healing-playwright-tests/src/index.ts +++ b/examples/self-healing-playwright-tests/src/index.ts @@ -4,15 +4,18 @@ export { runHealingTest } from './healer.js'; export { route, loadRouterConfig } from './router.js'; -export { runInSandbox } from './runner.js'; +export { classifyFailure, runInSandbox } from './runner.js'; +export type { RunOptions } from './runner.js'; export type { - TestSpec, + FailureType, GeneratedTest, - RunResult, HealAttempt, HealingTrace, - RouterConfig, ProviderName, + RouterConfig, + RunPhase, + RunResult, + TestSpec, } from './types.js'; export { AllProvidersFailedError } from './types.js'; diff --git a/examples/self-healing-playwright-tests/src/runner.ts b/examples/self-healing-playwright-tests/src/runner.ts index 9d0f0a02..9151d388 100644 --- a/examples/self-healing-playwright-tests/src/runner.ts +++ b/examples/self-healing-playwright-tests/src/runner.ts @@ -11,7 +11,7 @@ import { CommandExitError, Sandbox } from '@e2b/code-interpreter'; -import type { GeneratedTest, RunResult } from './types.js'; +import type { GeneratedTest, RunPhase, RunResult } from './types.js'; const TEMPLATE = 'playwright-chromium'; @@ -31,12 +31,83 @@ const SNAPSHOT_PATH = `${WORK_DIR}/failure.html`; // "chrome-headless-shell-<rev>: cannot execute binary" type failures. const PLAYWRIGHT_VERSION = '1.51.1'; +// Sentinels used to bracket the injected page-snapshot hook so we can +// strip-and-reattach it on every heal pass. Without strip-and-reattach, +// successive heals either: +// (a) drop the hook entirely → no snapshot → healing degrades to +// guessing from stderr; or +// (b) duplicate the hook → "Identifier '_fs' has already been +// declared" SyntaxError → every retry fails before the test runs. +// Battle-tested at qualitymax.io after both modes bit us in production. +const HOOK_BEGIN = '// --- begin self-healing capture hook ---'; +const HOOK_END = '// --- end self-healing capture hook ---'; + +// We inline `require('fs')` inside the callback rather than adding a +// top-level `import fs from 'fs'`. Two reasons: +// 1. The LLM-generated code may or may not already import fs — adding +// our own import risks a duplicate binding. +// 2. If the LLM accidentally pastes the hook twice during a heal, two +// top-level `_fs` bindings produce a SyntaxError that's invisible +// until the next run. `require()` is cached and binding-free, so +// duplication is harmless (the strip-and-reattach still runs). +const CAPTURE_HOOK = `${HOOK_BEGIN} +test.afterEach(async ({ page }, testInfo) => { + if (testInfo.status !== testInfo.expectedStatus) { + try { + const _html = await page.content(); + // eslint-disable-next-line @typescript-eslint/no-require-imports + require('fs').writeFileSync('${SNAPSHOT_PATH}', _html.substring(0, 60000)); + } catch (_e) { + // Best-effort: a snapshot failure must never mask the real test failure. + } + } +}); +${HOOK_END}`; + +const HOOK_STRIP_RE = new RegExp( + `\\n*${HOOK_BEGIN.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&')}[\\s\\S]*?${HOOK_END.replace( + /[.*+?^${}()|[\\]\\\\]/g, + '\\\\$&', + )}\\n*`, + 'g', +); + +/** + * Remove a previously injected capture hook (idempotent). + * Always strip then reattach on each heal — see HOOK_BEGIN comment. + */ +function stripCaptureHook(code: string): string { + return code.includes(HOOK_BEGIN) ? code.replace(HOOK_STRIP_RE, '\n') : code; +} + +/** + * Append our controlled capture hook to the LLM-generated test. + * + * We do this server-side instead of asking the LLM to write the snapshot + * itself (the obvious approach). Battle-tested reason: when the LLM is + * responsible for the snapshot path, you accumulate three failure modes + * none of which fail loudly: + * + * 1. LLM forgets the hook in the heal pass. + * 2. LLM writes to a root-owned directory (`/app/...`) → EACCES, + * silent: the test fails for unrelated reasons and you have no DOM. + * 3. LLM duplicates `import fs from 'fs'` on heal → SyntaxError. + * + * Injecting the hook ourselves makes the snapshot a guaranteed side-effect + * of any failure, not a thing the agent might or might not remember to do. + */ +function injectCaptureHook(code: string): string { + return `${stripCaptureHook(code).trimEnd()}\n\n${CAPTURE_HOOK}\n`; +} + +export const _internals = { stripCaptureHook, injectCaptureHook, CAPTURE_HOOK }; + /** * Write the generated test into the sandbox along with a small Playwright * config that drops the test runner into the working directory. */ async function prepareSandbox(sandbox: Sandbox, code: string): Promise<void> { - await sandbox.files.write(TEST_FILE_PATH, code); + await sandbox.files.write(TEST_FILE_PATH, injectCaptureHook(code)); await sandbox.files.write( `${WORK_DIR}/playwright.config.ts`, `import { defineConfig } from '@playwright/test'; @@ -69,19 +140,58 @@ async function ensureTestRunner(sandbox: Sandbox): Promise<void> { } /** - * Best-effort capture of the page state at failure time. Reads the - * snapshot file if the test wrote one (see HEALING_INSTRUCTIONS in - * healer.ts which asks the LLM to dump page.content() on failure). + * Best-effort read of the page snapshot the injected hook writes on failure. + * + * We control where the snapshot lives (`SNAPSHOT_PATH`) and we control the + * hook that writes it, so the only reasons the file is missing are: + * - the test crashed before page.content() resolved (e.g. browser launch + * failure); or + * - the failing fixture was so broken Playwright never reached afterEach. + * In both cases we degrade gracefully and let the healer work from stderr. */ async function tryReadSnapshot(sandbox: Sandbox): Promise<string | undefined> { try { const content = await sandbox.files.read(SNAPSHOT_PATH); - return typeof content === 'string' ? content : undefined; + return typeof content === 'string' && content.length > 0 ? content : undefined; } catch { return undefined; } } +export interface RunOptions { + /** Optional progress callback fired on each in-sandbox phase boundary. */ + onProgress?: (phase: RunPhase, detail?: string) => void; + /** Optional callback for raw stdout chunks as the sandbox emits them. */ + onStdoutChunk?: (chunk: string) => void; + /** Optional callback for raw stderr chunks as the sandbox emits them. */ + onStderrChunk?: (chunk: string) => void; +} + +/** + * Parse phase markers out of streamed output. + * + * The in-sandbox runner echoes `QMAX_PHASE:<phase>` lines at known + * boundaries; we surface those to the optional `onProgress` callback so + * UI consumers can show "Installing @playwright/test…", "Running test…" + * etc. without parsing arbitrary npm/Playwright output. The convention + * is borrowed from qualitymax.io's progress bar — a sentinel grep + * survives version bumps of npm/playwright that would break a regex on + * their normal log output. + */ +const PHASE_RE = /QMAX_PHASE:([a-z_]+)/g; + +function parsePhases( + chunk: string, + onProgress?: (phase: RunPhase, detail?: string) => void, +): void { + if (!onProgress) return; + let match: RegExpExecArray | null; + PHASE_RE.lastIndex = 0; + while ((match = PHASE_RE.exec(chunk)) !== null) { + onProgress(match[1] as RunPhase); + } +} + /** * Run a generated test once in a fresh sandbox. * @@ -90,11 +200,15 @@ async function tryReadSnapshot(sandbox: Sandbox): Promise<string | undefined> { */ export async function runInSandbox( generated: GeneratedTest, + opts: RunOptions = {}, ): Promise<RunResult> { + opts.onProgress?.('sandbox_starting'); const sandbox = await Sandbox.create(TEMPLATE); try { await prepareSandbox(sandbox, generated.code); + opts.onProgress?.('project_uploaded'); await ensureTestRunner(sandbox); + opts.onProgress?.('deps_installed'); // E2B's SDK throws CommandExitError on non-zero exit rather than // returning a result. For a test runner that's the wrong default -- @@ -105,12 +219,30 @@ export async function runInSandbox( // // PLAYWRIGHT_BROWSERS_PATH points at the template's pre-installed // Chromium so @playwright/test doesn't try to download its own. + // + // The `echo QMAX_PHASE:test_started` markers are picked up by + // `parsePhases` and forwarded to onProgress -- see the comment on + // PHASE_RE for why we use sentinels instead of parsing npm output. + const cmd = + `echo QMAX_PHASE:test_started && ` + + `PLAYWRIGHT_BROWSERS_PATH=${BROWSERS_PATH} ` + + `npx playwright test --reporter=list 2>&1; ` + + `status=$?; echo QMAX_PHASE:test_finished; exit $status`; + let exec: { exitCode: number; stdout: string; stderr: string }; try { - exec = await sandbox.commands.run( - `PLAYWRIGHT_BROWSERS_PATH=${BROWSERS_PATH} npx playwright test --reporter=list 2>&1`, - { cwd: WORK_DIR, timeoutMs: 5 * 60 * 1000 }, - ); + exec = await sandbox.commands.run(cmd, { + cwd: WORK_DIR, + timeoutMs: 5 * 60 * 1000, + onStdout: (chunk: string) => { + opts.onStdoutChunk?.(chunk); + parsePhases(chunk, opts.onProgress); + }, + onStderr: (chunk: string) => { + opts.onStderrChunk?.(chunk); + parsePhases(chunk, opts.onProgress); + }, + }); } catch (err) { if (!(err instanceof CommandExitError)) throw err; exec = err; @@ -118,6 +250,7 @@ export async function runInSandbox( const passed = exec.exitCode === 0; const failureSnapshot = passed ? undefined : await tryReadSnapshot(sandbox); + opts.onProgress?.('artifacts_collected'); return { passed, @@ -125,8 +258,40 @@ export async function runInSandbox( stdout: exec.stdout, stderr: exec.stderr, failureSnapshot, + failureType: passed ? undefined : classifyFailure(exec.stdout, exec.stderr), }; } finally { await sandbox.kill(); } } + +/** + * Classify a Playwright failure into one of a small set of buckets. + * + * The buckets are tuned for what the healer can actually act on: + * - `strict_mode_violation` → tell the LLM how to disambiguate + * (anchor by unique text, scope by parent role, use data-testid). + * - `locator_not_found` → broaden the selector strategy. + * - `timeout` → wait for network/load state, not bare element. + * - `assertion_failed` → likely the expected value changed, not the + * locator; tell the LLM to use a less specific assertion. + * + * Battle-tested ordering: strict-mode check before "locator …" because + * strict-mode errors *contain* "locator" in their message and would + * otherwise be mis-bucketed. + */ +export function classifyFailure( + stdout: string, + stderr: string, +): RunResult['failureType'] { + const text = `${stdout}\n${stderr}`.toLowerCase(); + if (/strict mode violation.*resolved to \d+ elements?/.test(text)) { + return 'strict_mode_violation'; + } + if (/locator|selector|waiting for/.test(text)) { + return 'locator_not_found'; + } + if (/timeout/.test(text)) return 'timeout'; + if (/assertion|expect\(/.test(text)) return 'assertion_failed'; + return 'unknown'; +} diff --git a/examples/self-healing-playwright-tests/src/types.ts b/examples/self-healing-playwright-tests/src/types.ts index dab55de2..6d06c6ac 100644 --- a/examples/self-healing-playwright-tests/src/types.ts +++ b/examples/self-healing-playwright-tests/src/types.ts @@ -11,6 +11,32 @@ export type ProviderName = 'anthropic' | 'openai' | 'google'; +/** + * Coarse classification of the failure so the healer can steer the next + * prompt. Battle-tested at qualitymax.io: without this, the healer treats + * "selector matches 4 elements" and "selector matches 0 elements" the + * same way and keeps emitting equivalent locators. + */ +export type FailureType = + | 'strict_mode_violation' + | 'locator_not_found' + | 'timeout' + | 'assertion_failed' + | 'unknown'; + +/** + * Sentinel phase emitted by the in-sandbox runner script. Surfaced via + * the optional `onProgress` callback on `runInSandbox` so callers driving + * a UI / CI can show real progress instead of an opaque "running…". + */ +export type RunPhase = + | 'sandbox_starting' + | 'project_uploaded' + | 'deps_installed' + | 'test_started' + | 'test_finished' + | 'artifacts_collected'; + export interface TestSpec { /** Natural-language description of what the test should do. */ description: string; @@ -36,6 +62,8 @@ export interface RunResult { stderr: string; /** Snapshot of the failing page (HTML), if the runner managed to capture one. */ failureSnapshot?: string; + /** Coarse classification of the failure, populated when !passed. */ + failureType?: FailureType; } export interface HealAttempt {