Improve Kernel browser execution with Browser Loop

agent/subagents/worker/agent.ts

@@ -4,6 +4,9 @@ import { getModelSettings } from "@/lib/model-config";
 import { taskCompletionSchema } from "@/lib/task-completion";
 
 export default defineAgent({
+  build: {
+    externalDependencies: ["@onkernel/browser-loop"],
+  },
   description:
     "Execute one bounded browser assignment for the root coordinator, including secure vault autofill, transaction preparation, optional durable browser images, human-takeover handoff, cleanup, and a concise verified result. Every initial and resumed call must include the task-completion outputSchema required by the root instructions.",
   model: defineDynamic({

agent/subagents/worker/instructions.md

@@ -21,7 +21,7 @@ You are `worker`, the root coordinator's dedicated browser executor. Complete on
 
 # Execution
 
-- Load the `browser-execution` skill for every browser assignment and use only `manage_browsers`, `execute_playwright_code`, `computer_action`, `capture_browser_image`, `list_vault`, and `fill_from_vault` as needed.
+- Load the `browser-execution` skill for every browser assignment and use only `manage_browsers`, `browser_snapshot`, `browser_act`, `execute_playwright_code`, `computer_action`, `capture_browser_image`, `list_vault`, and `fill_from_vault` as needed.
 - Keep ordinary `computer_action` screenshots temporary and model-visible only. Use `capture_browser_image` only when the assignment requests an image or visual evidence materially improves the final result. Never persist routine debugging screenshots. Return only image descriptors actually produced by that tool.
 - Create one browser and reuse it. When the assignment includes the target URL, pass it as `start_url` during creation instead of spending a separate browser call on the initial navigation. Persist through recoverable failures, but use at most two materially different tactics for a blocked state. Respect the assignment's bounds, active cancellation, and the browser tool's time limits.
 - Re-read the page after coordinator-approved continuation or human takeover because the browser state may have changed.

agent/subagents/worker/lib/vault-screenshot-mask.ts

@@ -15,6 +15,21 @@ export async function withVaultScreenshotMask<T>(
   }
 }
 
+export async function withVaultBrowserObservationMask<T>(
+  sessionId: string,
+  signal: AbortSignal | undefined,
+  observe: () => Promise<T>
+) {
+  await setVaultAccessibilityMask(sessionId, "add", signal);
+  try {
+    return await observe();
+  } finally {
+    await setVaultAccessibilityMask(sessionId, "remove", undefined).catch(
+      () => undefined
+    );
+  }
+}
+
 async function setVaultScreenshotMask(
   sessionId: string,
   action: "add" | "remove",
@@ -29,30 +44,76 @@ async function setVaultScreenshotMask(
         if (existing) {
           const refs = Number.parseInt(existing.dataset.vaultMaskRefs || "0", 10);
           existing.dataset.vaultMaskRefs = String((Number.isFinite(refs) ? refs : 0) + 1);
-          return;
-        }
-        const style = document.createElement("style");
-        style.id = styleId;
-        style.dataset.vaultMaskRefs = "1";
-        style.textContent = selector + " { color: transparent !important; text-shadow: 0 0 8px black !important; -webkit-text-security: disc !important; }";
-        document.documentElement.append(style);`
+        } else {
+          const style = document.createElement("style");
+          style.id = styleId;
+          style.dataset.vaultMaskRefs = "1";
+          style.textContent = selector + " { color: transparent !important; text-shadow: 0 0 8px black !important; -webkit-text-security: disc !important; }";
+          document.documentElement.append(style);
+        }`
       : `
         const style = document.getElementById(styleId);
-        if (!style) return;
-        const refs = Number.parseInt(style.dataset.vaultMaskRefs || "1", 10);
-        const remainingRefs = Math.max(0, (Number.isFinite(refs) ? refs : 1) - 1);
-        if (remainingRefs > 0) {
-          style.dataset.vaultMaskRefs = String(remainingRefs);
-        } else {
-          style.remove();
+        if (style) {
+          const refs = Number.parseInt(style.dataset.vaultMaskRefs || "1", 10);
+          const remainingRefs = Math.max(0, (Number.isFinite(refs) ? refs : 1) - 1);
+          if (remainingRefs > 0) {
+            style.dataset.vaultMaskRefs = String(remainingRefs);
+          } else {
+            style.remove();
+          }
+        }`;
+  await runMaskOperation(
+    sessionId,
+    operation,
+    { selector, styleId },
+    action,
+    signal
+  );
+}
+
+async function setVaultAccessibilityMask(
+  sessionId: string,
+  action: "add" | "remove",
+  signal?: AbortSignal
+) {
+  const selector = '[data-vault-secret="true"]';
+  const operation =
+    action === "add"
+      ? `
+        for (const element of document.querySelectorAll(selector)) {
+          const stalePrevious = element.dataset.vaultPreviousAriaHidden;
+          if (stalePrevious === "__absent__") element.removeAttribute("aria-hidden");
+          else if (stalePrevious !== undefined) element.setAttribute("aria-hidden", stalePrevious);
+          element.dataset.vaultPreviousAriaHidden = element.hasAttribute("aria-hidden")
+            ? element.getAttribute("aria-hidden") || ""
+            : "__absent__";
+          element.setAttribute("aria-hidden", "true");
+        }`
+      : `
+        for (const element of document.querySelectorAll(selector)) {
+          const previous = element.dataset.vaultPreviousAriaHidden;
+          if (previous === "__absent__") element.removeAttribute("aria-hidden");
+          else if (previous !== undefined) element.setAttribute("aria-hidden", previous);
+          delete element.dataset.vaultPreviousAriaHidden;
         }`;
+  await runMaskOperation(sessionId, operation, { selector }, action, signal);
+}
+
+async function runMaskOperation(
+  sessionId: string,
+  operation: string,
+  parameters: Record<string, string>,
+  action: "add" | "remove",
+  signal?: AbortSignal
+) {
   const code = `
 for (const currentContext of browser.contexts()) {
   for (const currentPage of currentContext.pages()) {
     for (const frame of currentPage.frames()) {
-      await frame.evaluate(({ styleId, selector }) => {
+      await frame.evaluate((parameters) => {
+        const { selector, styleId } = parameters;
         ${operation}
-      }, ${JSON.stringify({ selector, styleId })}).catch(() => undefined);
+      }, ${JSON.stringify(parameters)}).catch(() => undefined);
     }
   }
 }
@@ -65,8 +126,8 @@ return true;`;
   if (!result.success) {
     throw new Error(
       action === "add"
-        ? "Vault fields could not be masked for screenshot capture."
-        : "Vault screenshot masking could not be removed."
+        ? "Vault fields could not be masked for browser observation."
+        : "Vault browser masking could not be removed."
     );
   }
 }

agent/subagents/worker/skills/browser-execution/SKILL.md

@@ -8,10 +8,10 @@ description: Complete a direct browser task, including recovery from blocked sit
 - Source of truth: use Kernel's current documentation directly for [browser sessions](https://kernel.sh/docs/introduction/create), [Playwright execution](https://kernel.sh/docs/browsers/playwright-execution), [computer controls](https://kernel.sh/docs/browsers/computer-controls), [stealth and CAPTCHA solving](https://kernel.sh/docs/browsers/bot-detection/stealth), and [live view](https://kernel.sh/docs/browsers/live-view). Do not infer a Kernel API shape from memory.
 - Browser execution is not web search. Never open a general search engine or browse search-result pages. Begin only with a known target site and an outcome that requires interaction, browser state, or visual inspection. If preliminary public discovery is missing, stop before creating a browser and return that routing blocker so the coordinator can use `web_search`.
 - Start work immediately. For a fresh assignment, create a browser directly instead of listing old sessions first, and pass the known target URL as `start_url` to avoid a separate initial navigation. Reuse the returned session ID for the rest of the assignment. Call `get_browser_live_view` only when the user explicitly asks for browser access.
-- Create one read-only browser and reuse it for the full job. Multiple read-only sessions can safely share the workspace profile. Immediately before a saved login is needed, note the current URL, delete that task browser, and create a replacement at the same URL with `save_changes: true`. Only one writable session can exist for the workspace; delete it as soon as authentication succeeds so Kernel saves the new login state. Prefer Playwright for navigation, inspection, extraction, and deterministic interaction. Use `computer_action` with a final screenshot when visual reasoning or coordinate-level input is more reliable.
+- Create one read-only browser and reuse it for the full job. Multiple read-only sessions can safely share the workspace profile. Immediately before a saved login is needed, note the current URL, delete that task browser, and create a replacement at the same URL with `save_changes: true`. Only one writable session can exist for the workspace; delete it as soon as authentication succeeds so Kernel saves the new login state. Prefer `browser_snapshot` plus `browser_act` for routine accessible-page interaction: use only current returned refs, attach semantic expectations to consequential steps, and re-snapshot after navigation or a stale-ref error. `browser_act` is omitted automatically when the selected model rejects its large schema; use Playwright with the snapshot evidence instead. Prefer Playwright for precise extraction, navigation, or interactions the ref surface cannot express. Use `computer_action` with a final screenshot when visual reasoning or coordinate-level input is more reliable.
 - A `computer_action` screenshot is temporary inspection data. Use `capture_browser_image` only for a screenshot or page image the parent may send to the user. When the assignment asks for a photo, pic, or image of a specific item, product, dish, or listing, prefer `source: "image_resource"` on the matching visible image element so the user receives the original page image. Use a viewport, full-page, or rendered-element screenshot only when the assignment explicitly requests screen or page context, no suitable image element exists, or original-resource capture falls back automatically. Use durable capture when the user requested an image or one image materially improves the final result, never for routine debugging. Keep every returned image descriptor for `final_output.images`; do not invent artifact ids or URLs.
 - Choose the smallest useful visual set. Return one image for a single result, or two to four distinct images only when a compact gallery materially helps the user compare visually differentiated options or verify an outcome. Useful cases include clothing, food, or product choices and a purchased item or safe, non-sensitive confirmation state. Every image must add distinct value; skip decorative, repetitive, text-only, or sensitive content.
-- Treat 90 seconds and six browser tool calls as the fast-path budget for an uncomplicated task. Each Playwright call should normally inspect the current state, perform all related safe actions, verify the resulting state, and return one compact object. Re-enter the model only for a meaningful page transition, an unknown state, approval, or recovery. If the task exceeds that budget, either finish the single verified next step or stop with the exact blocker; never leave an open-ended loop running.
+- Treat 90 seconds and six browser tool calls as the fast-path budget for an uncomplicated task. Each `browser_act` or Playwright call should normally perform all related safe actions, verify the resulting state, and return one compact result. Re-enter the model only for a meaningful page transition, an unknown state, approval, or recovery. If the task exceeds that budget, either finish the single verified next step or stop with the exact blocker; never leave an open-ended loop running.
 - Use names, email addresses, phone numbers, mailing addresses, and other non-credential form values directly when the coordinator provides them in the task. Do not require those values to be saved in the vault first.
 - When a page needs a saved login, payment card, or address, call `list_vault`, choose the correct opaque handle, focus one visible control in the intended form, then pass only its handle and the browser session ID to `fill_from_vault`. Login fill is restricted to the saved origin and focused login form; on a multi-step login, advance the page and call `fill_from_vault` again for the next visible credential field. Never pass vault fields, selectors, origins, or secret values. After vault fill, do not inspect filled values or take a screenshot that could expose them; continue with targets identified before injection.
 - If secure fill fails, report the exact tool error and last verified page state. Never infer a cross-origin or provider limitation solely from the page layout.