Build operator observability dashboard and telemetry

CONTRIBUTING.md

@@ -163,6 +163,17 @@ logs. Update this guide when architecture, ownership or generated contracts chan
 Comments and documentation are English. Reuse existing helpers and error mapping;
 split oversized components before extending them. Use `internal/obs/log` for logs.
 
+Operator observability is a side-channel owned by Core. The authenticated
+`/core/v1/admin/observability` reads bounded request, terminal Turn,
+tool attempt, and collector aggregates; the Core Web uses it through
+`packages/agents-client`'s `AdminClient`, alongside the existing Core service
+metrics endpoint. Runtime samples retain their separate public history
+contract under the `observability_runtime_samples` table. Native model attempts
+have a reserved table but no qualified producer; configured models, cumulative
+Usage, and Turns must not be counted as model calls. Missing collector coverage
+stays unavailable rather than becoming zero. See
+`contracts/agents-api/system-observability-plan.md` for source and retention details.
+
 ## Required checks
 
 Run `make check` before completion. The standalone gate includes all daemon/shared

apps/web/e2e/agents-lifecycle.spec.ts

@@ -210,6 +210,48 @@ test("opens Dashboard as the default landing page", async ({ page, request }) =>
   await expect(page.locator(".dashboard-page")).toBeVisible();
 });
 
+test("shows operator tables and keeps incomplete request coverage unavailable", async ({ page, request }, testInfo) => {
+  await resetFixture(request);
+  await page.route("**/console/config", (route) => route.fulfill({ json: {
+    sandbox_admin: true, node_installer: false, node_installer_sha256: "",
+  } }));
+  const end = new Date(Math.floor(Date.now() / 60_000) * 60_000);
+  const start = new Date(end.getTime() - 60 * 60_000);
+  await page.route("**/core/v1/admin/observability?range=1h", (route) => route.fulfill({ json: {
+    generated_at: end.toISOString(), start: start.toISOString(), end: end.toISOString(), step_seconds: 60,
+    requests: [{ start: start.toISOString(), route_family: "sessions", outcome: "server_error", count: 2,
+      latency_sum_ms: 120, latency_bucket_counts: [0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] }],
+    collector: [{ start: start.toISOString(), source: "request", attempted_count: 2, observed_count: 2,
+      unavailable_count: 0, timeout_count: 0, dropped_count: 0, export_failed_count: 0 }],
+    turns: [{ start: start.toISOString(), status: "failed", count: 1, queue_p95_ms: 50, execution_p95_ms: 400 }],
+    tools: [{ start: start.toISOString(), category: "command", outcome: "error", count: 1,
+      timed_count: 1, duration_p95_ms: 300 }],
+  } }));
+  await page.route("**/core/v1/admin/core-metrics?range=1h", (route) => route.fulfill({ json: {
+    object: "core.metrics", range: { start: start.toISOString(), end: end.toISOString(), resolution_seconds: 60 },
+    service: { status: "healthy", revision: null, started_at: start.toISOString(), execution_owner: true },
+    execution: { slots_in_use: 1, slots_total: 4, queued_turns: 3, waiting_for_daemon: 1,
+      in_progress_turns: 2, oldest_queued_seconds: 12, connected_daemons: 2, interrupted: 0,
+      unavailable: 0, queue_wait_ms: { p50: 20, p95: 50 }, series: [] },
+    database: { ping_ms: { p50: 3, p95: 7 }, pool: { in_use: 2, idle: 4, max: 20 }, size_bytes: 1024, series: [] },
+    jobs: [{ id: "scheduler", status: "running", last_run_at: end.toISOString(), processed: 5, failed: 0 }],
+    process: { memory_bytes: 1048576, goroutines: 42 },
+  } }));
+  await page.goto("/");
+  await page.getByRole("tab", { name: "Observability" }).click();
+  const system = page.getByRole("region", { name: "System observability" });
+  await expect(system.getByRole("heading", { name: "System observability" })).toBeVisible();
+  await expect(system.getByRole("heading", { name: "Terminal Turns" })).toBeVisible();
+  await expect(system.getByRole("heading", { name: "Actual model invocations" })).toBeVisible();
+  await expect(system.getByRole("heading", { name: "Sandbox nodes" })).toBeVisible();
+  await expect(page.getByRole("heading", { name: "Core service" })).toBeVisible();
+  await expect(page.getByRole("heading", { name: "Background jobs" })).toBeVisible();
+  await expect(system.getByText("Unavailable", { exact: true }).first()).toBeVisible();
+  await expect(system.getByRole("rowheader", { name: "sessions" })).toBeVisible();
+  await expect(system.getByRole("rowheader", { name: "command" })).toBeVisible();
+  await attachElementScreenshot(page.locator(".dashboard-page"), testInfo, "system-observability-dashboard");
+});
+
 test("explains a 502 Core backend failure and opens copyable Docker recovery steps", async ({ page }) => {
   await page.route("**/v1/agents**", async (route) => {
     await route.fulfill({
@@ -2388,9 +2430,8 @@ test("presents Dashboard page-chain results and System boundaries without extra
   await expect(dashboard).toContainText("Latest complete paginated reads");
   await expect(dashboard.locator(".dashboard-summary > div").filter({ hasText: "Agents" })).toContainText("3");
   await expect(dashboard.locator(".dashboard-summary > div").filter({ hasText: "Sessions" })).toContainText("1");
-  await expect(dashboard.locator(".dashboard-summary > div").filter({ hasText: "In progress" })).toContainText("0");
-  await expect(dashboard.locator(".dashboard-summary > div").filter({ hasText: "Needs attention" })).toContainText("0");
-  await expect(dashboard).not.toContainText("Reported aggregate tokens");
+  await expect(dashboard.locator(".dashboard-summary > div").filter({ hasText: "Active sandboxes" })).toBeVisible();
+  await expect(dashboard.locator(".dashboard-summary > div").filter({ hasText: "Reported tokens" })).toBeVisible();
   await expect(dashboard).toContainText("No Sessions currently need attention.");
   await expect(dashboard.getByRole("button", { name: /Create agent/ })).toBeVisible();
   await expect(dashboard.getByRole("button", { name: /Start session/ })).toBeVisible();
@@ -2683,7 +2724,13 @@ test("renders Runtime telemetry as visual snapshot panels with details on demand
   await page.goto("/");
   const dashboard = page.locator(".dashboard-page");
   await expect(dashboard.getByRole("heading", { name: "Dashboard", exact: true })).toBeVisible();
+  await dashboard.getByRole("tab", { name: "Observability" }).click();
   const refresh = dashboard.getByRole("button", { name: "Refresh Dashboard snapshot" });
+  const sandboxDiagnostics = dashboard.getByRole("region", { name: "Sandbox diagnostics" });
+  await expect(sandboxDiagnostics).toBeVisible();
+  await expect(sandboxDiagnostics).toContainText("12 managed targets");
+  await expect(sandboxDiagnostics).toContainText("12 with measured usage and limit");
+  await expect(sandboxDiagnostics).toContainText("Highest memory pressure");
   await expect(dashboard.locator(".dashboard-runtime-sample-count")).toContainText("1 sample ·");
   await expect(dashboard.getByRole("heading", { name: "CPU usage" })).toBeVisible();
   await expect(dashboard.getByRole("heading", { name: "Memory usage" })).toBeVisible();
@@ -2735,7 +2782,7 @@ test("renders Runtime telemetry as visual snapshot panels with details on demand
   await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toBeVisible();
   await cpuChart.click({ position: { x: 260, y: 90 } });
   await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Pinned");
-  await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("10%");
+  await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Unavailable");
   await cpuChart.focus();
   await cpuChart.press("ArrowRight");
   await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Pinned");
@@ -2913,6 +2960,7 @@ test("restores retained Runtime history after a Dashboard reload", async ({ page
   });
 
   await page.goto("/");
+  await page.getByRole("tab", { name: "Observability" }).click();
   const dashboard = page.locator(".dashboard-runtime-panel");
   await expect(dashboard.getByRole("group", { name: "Runtime trend source" })).toHaveCount(0);
   await expect(dashboard.getByLabel(/Durable · 30s; 1 Runtime targets/)).toBeVisible();
@@ -2930,9 +2978,9 @@ test("restores retained Runtime history after a Dashboard reload", async ({ page
   const durableCpuCard = durableCpuChart.locator("xpath=ancestor::section[contains(@class, 'dashboard-runtime-trend-card')]");
   await durableCpuChart.focus();
   await durableCpuChart.press("ArrowLeft");
-  await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("0%");
+  await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Unavailable");
   await durableCpuChart.press("ArrowLeft");
-  await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("0%");
+  await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Unavailable");
   const durableMemoryCard = dashboard.getByRole("region", { name: "Memory usage durable history chart" });
   const durableMemorySpan = await durableMemoryCard.locator("canvas").evaluate((canvas: HTMLCanvasElement) => {
     const context = canvas.getContext("2d");
@@ -2973,6 +3021,7 @@ test("restores retained Runtime history after a Dashboard reload", async ({ page
   await expect(refreshedDurableCpuChart).toHaveAttribute("data-view-end", String(durableZoomEnd));
 
   await page.reload();
+  await page.getByRole("tab", { name: "Observability" }).click();
   await expect(dashboard.getByRole("group", { name: "Runtime trend source" })).toHaveCount(0);
   await expect(dashboard.getByLabel(/Durable · 30s; 1 Runtime targets/)).toBeVisible();
   await expect(dashboard.getByText("CPU usage durable trend available")).toBeAttached();

apps/web/e2e/sandbox-setup.spec.ts

@@ -189,13 +189,15 @@ for (const operation of ["setup", "enrollment"] as const) {
 }
 
 test("unpaired or unavailable consoles show setup guidance without admin credentials or manager requests", async ({ page, request }) => {
+  await expect.poll(async () => ((await (await request.get(`${fixture}/__fixture/sandbox`)).json()) as { calls: unknown[] }).calls.length).toBeGreaterThan(0);
+  const initialCalls = ((await (await request.get(`${fixture}/__fixture/sandbox`)).json()) as { calls: unknown[] }).calls.length;
   for (const body of [{ sandbox_admin: false, node_installer: true }, null]) {
     await page.route("**/console/config", (route) => route.fulfill(body ? { contentType: "application/json", body: JSON.stringify(body) } : { status: 404, body: "Not found" }));
     await page.reload();
     await page.getByRole("button", { name: "Hosted Sandbox Manager", exact: true }).click();
     await expect(page.getByRole("alert")).toContainText("Sandbox administration is not configured");
     await expect(page.getByLabel("Deployment admin key")).toHaveCount(0);
-    expect((await (await request.get(`${fixture}/__fixture/sandbox`)).json()).calls).toHaveLength(0);
+    expect((await (await request.get(`${fixture}/__fixture/sandbox`)).json()).calls).toHaveLength(initialCalls);
   }
   await page.route("**/console/config", (route) => route.fulfill({ contentType: "application/json", body: JSON.stringify({ sandbox_admin: true, node_installer: true, node_installer_sha256: "a".repeat(64) }) }));
   await page.getByRole("button", { name: "Refresh sandbox state" }).click();

apps/web/src/App.tsx

@@ -2367,6 +2367,7 @@ export function App() {
               runtimeCollectionError={runtimeCollectionError}
               runtimeCollectionHasSnapshot={runtimeCollectionHasSnapshot}
               loadRuntimeHistory={loadDashboardRuntimeHistory}
+              coreBaseUrl={connection.baseUrl}
               onRefresh={refreshDashboard}
               onCreateAgent={openAgentSetup}
               onStartSession={() => openSessionSetup()}

apps/web/src/features/dashboard/CoreServiceMetricsContent.tsx

@@ -0,0 +1,70 @@
+import { useEffect, useState } from "react";
+import { useTranslation } from "react-i18next";
+import { AdminClient, type CoreMetricsView, type OperatorMetricsRange } from "@agents-core-web/agents-client";
+
+import { isLocalProxyBaseUrl } from "../../lib/connection";
+import { formatDashboardBytes } from "./dashboard-model";
+
+export function CoreServiceMetricsContent({ coreBaseUrl, range, refresh }: { coreBaseUrl: string; range: OperatorMetricsRange; refresh: number }) {
+  const { t, i18n } = useTranslation("dashboard");
+  const localSource = isLocalProxyBaseUrl(coreBaseUrl);
+  const [metrics, setMetrics] = useState<CoreMetricsView | null>(null);
+  const [failed, setFailed] = useState(false);
+  useEffect(() => {
+    const controller = new AbortController();
+    if (!localSource) {
+      setMetrics(null);
+      setFailed(false);
+      return () => controller.abort();
+    }
+    setMetrics(null);
+    setFailed(false);
+    void new AdminClient().retrieveCoreMetrics(range, { signal: controller.signal }).then((value) => {
+      if (!controller.signal.aborted) { setMetrics(value); setFailed(false); }
+    }).catch(() => {
+      if (!controller.signal.aborted) { setMetrics(null); setFailed(true); }
+    });
+    return () => controller.abort();
+  }, [localSource, range, refresh]);
+  const unavailable = t("system.unavailableValue");
+  const number = (value: number | null | undefined) => value == null ? unavailable : value.toLocaleString(i18n.resolvedLanguage);
+  const milliseconds = (value: number | null | undefined) => value == null ? unavailable : `${number(Math.round(value))} ms`;
+  const execution = metrics?.execution;
+  const database = metrics?.database;
+  const process = metrics?.process;
+  const rows: Array<[string, string]> = [
+    [t("core.queued"), number(execution?.queued_turns)],
+    [t("core.running"), number(execution?.in_progress_turns)],
+    [t("core.waitingDaemon"), number(execution?.waiting_for_daemon)],
+    [t("core.connectedDaemons"), number(execution?.connected_daemons)],
+    [t("core.oldestQueue"), execution?.oldest_queued_seconds == null ? unavailable : `${number(Math.round(execution.oldest_queued_seconds))} s`],
+    [t("core.queueP95"), milliseconds(execution?.queue_wait_ms.p95)],
+    [t("core.interrupted"), number(execution?.interrupted)],
+    [t("core.unavailable"), number(execution?.unavailable)],
+  ];
+  return <section className="dashboard-system dashboard-core-metrics" aria-labelledby="dashboard-core-metrics-heading">
+    <header className="dashboard-system-header"><div>
+      <h2 id="dashboard-core-metrics-heading">{t("core.title")}</h2>
+      <p>{t("core.subtitle")}</p>
+    </div><span className="dashboard-system-source">{metrics ? t("core.status", { status: metrics.service.status }) : !localSource ? t("system.adminUnavailable") : failed ? t("system.stale") : t("system.loading")}</span></header>
+    <div className="dashboard-system-metrics">
+      <div><small>{t("core.slots")}</small><strong>{execution?.slots_in_use == null || execution.slots_total == null ? unavailable : `${number(execution.slots_in_use)}/${number(execution.slots_total)}`}</strong><span>{t("core.executionOwner", { status: metrics?.service.execution_owner == null ? unavailable : metrics.service.execution_owner ? t("core.yes") : t("core.no") })}</span></div>
+      <div><small>{t("core.queue")}</small><strong>{number(execution?.queued_turns)}</strong><span>{t("core.runningCount", { value: number(execution?.in_progress_turns) })}</span></div>
+      <div><small>{t("core.databasePing")}</small><strong>{milliseconds(database?.ping_ms.p95)}</strong><span>{t("core.pool", { used: number(database?.pool.in_use), max: number(database?.pool.max) })}</span></div>
+      <div><small>{t("core.processMemory")}</small><strong>{process?.memory_bytes == null ? unavailable : formatDashboardBytes(process.memory_bytes)}</strong><span>{t("core.goroutines", { value: number(process?.goroutines) })}</span></div>
+    </div>
+    <div className="dashboard-system-tables">
+      <section><h3>{t("core.executionTable")}</h3><div className="dashboard-system-table-scroll"><table><tbody>{rows.map(([label, value]) => <tr key={label}><th>{label}</th><td>{value}</td></tr>)}</tbody></table></div></section>
+      <section><h3>{t("core.databaseTable")}</h3><div className="dashboard-system-table-scroll"><table><tbody>
+        <tr><th>{t("core.pingP50")}</th><td>{milliseconds(database?.ping_ms.p50)}</td></tr>
+        <tr><th>{t("core.pingP95")}</th><td>{milliseconds(database?.ping_ms.p95)}</td></tr>
+        <tr><th>{t("core.poolInUse")}</th><td>{number(database?.pool.in_use)}</td></tr>
+        <tr><th>{t("core.poolIdle")}</th><td>{number(database?.pool.idle)}</td></tr>
+        <tr><th>{t("core.databaseSize")}</th><td>{database?.size_bytes == null ? unavailable : formatDashboardBytes(database.size_bytes)}</td></tr>
+      </tbody></table></div></section>
+      <section className="dashboard-system-nodes"><h3>{t("core.jobsTable")}</h3>
+        {metrics?.jobs.length ? <div className="dashboard-system-table-scroll"><table><thead><tr><th>{t("core.job")}</th><th>{t("core.jobStatus")}</th><th>{t("core.lastRun")}</th><th>{t("core.processed")}</th><th>{t("core.failed")}</th></tr></thead><tbody>{metrics.jobs.map((job) => <tr key={job.id}><th>{job.id}</th><td>{job.status}</td><td>{job.last_run_at == null ? unavailable : new Date(job.last_run_at).toLocaleString(i18n.resolvedLanguage)}</td><td>{number(job.processed)}</td><td>{number(job.failed)}</td></tr>)}</tbody></table></div> : <p>{!localSource ? t("system.adminUnavailable") : failed ? t("core.metricsUnavailable") : t("system.loading")}</p>}
+      </section>
+    </div>
+  </section>;
+}