Skip to content

Add Sandbox diagnostics and system observability plan #93

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
sam2tom wants to merge 1 commit into from
Closed

Add Sandbox diagnostics and system observability plan #93

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions apps/web/e2e/agents-lifecycle.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2684,6 +2684,11 @@ test("renders Runtime telemetry as visual snapshot panels with details on demand
const dashboard = page.locator(".dashboard-page");
await expect(dashboard.getByRole("heading", { name: "Dashboard", exact: true })).toBeVisible();
const refresh = dashboard.getByRole("button", { name: "Refresh Dashboard snapshot" });
const sandboxDiagnostics = dashboard.getByRole("region", { name: "Sandbox diagnostics" });
await expect(sandboxDiagnostics).toBeVisible();
await expect(sandboxDiagnostics).toContainText("12 managed targets");
await expect(sandboxDiagnostics).toContainText("12 with measured usage and limit");
await expect(sandboxDiagnostics).toContainText("Highest memory pressure");
await expect(dashboard.locator(".dashboard-runtime-sample-count")).toContainText("1 sample ·");
await expect(dashboard.getByRole("heading", { name: "CPU usage" })).toBeVisible();
await expect(dashboard.getByRole("heading", { name: "Memory usage" })).toBeVisible();
Expand Down Expand Up @@ -2735,7 +2740,7 @@ test("renders Runtime telemetry as visual snapshot panels with details on demand
await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toBeVisible();
await cpuChart.click({ position: { x: 260, y: 90 } });
await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Pinned");
await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("10%");
await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Unavailable");
await cpuChart.focus();
await cpuChart.press("ArrowRight");
await expect(cpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Pinned");
Expand Down Expand Up @@ -2930,9 +2935,9 @@ test("restores retained Runtime history after a Dashboard reload", async ({ page
const durableCpuCard = durableCpuChart.locator("xpath=ancestor::section[contains(@class, 'dashboard-runtime-trend-card')]");
await durableCpuChart.focus();
await durableCpuChart.press("ArrowLeft");
await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("0%");
await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Unavailable");
await durableCpuChart.press("ArrowLeft");
await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("0%");
await expect(durableCpuCard.locator(".dashboard-runtime-trend-tooltip")).toContainText("Unavailable");
const durableMemoryCard = dashboard.getByRole("region", { name: "Memory usage durable history chart" });
const durableMemorySpan = await durableMemoryCard.locator("canvas").evaluate((canvas: HTMLCanvasElement) => {
const context = canvas.getContext("2d");
Expand Down
58 changes: 58 additions & 0 deletions apps/web/src/features/dashboard/DashboardView.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -633,6 +633,64 @@
border-bottom: 1px solid var(--line);
}

.dashboard-sandbox-insights {
padding: 16px;
border-bottom: 1px solid var(--line);
}

.dashboard-sandbox-insights > header,
.dashboard-sandbox-diagnostics {
display: flex;
justify-content: space-between;
gap: 16px;
}

.dashboard-sandbox-insights h3,
.dashboard-sandbox-insights h4,
.dashboard-sandbox-insights p {
margin: 0;
}

.dashboard-sandbox-insights h3 { font-size: 14px; }
.dashboard-sandbox-insights h4 { font-size: 12px; }
.dashboard-sandbox-insights p,
.dashboard-sandbox-insights small { color: var(--fg-muted); font-size: 11px; }

.dashboard-sandbox-insight-grid {
display: grid;
grid-template-columns: repeat(4, minmax(0, 1fr));
gap: 8px;
margin-top: 12px;
}

.dashboard-sandbox-insight-grid > div {
display: grid;
gap: 3px;
padding: 10px 12px;
border: 1px solid var(--line);
border-radius: 8px;
min-width: 0;
}

.dashboard-sandbox-insight-grid strong {
font: 550 20px/1.2 var(--font-mono);
font-variant-numeric: tabular-nums;
}

.dashboard-sandbox-insight-grid span { color: var(--fg-muted); font-size: 10px; }
.dashboard-sandbox-diagnostics { margin-top: 14px; }
.dashboard-sandbox-diagnostics > div { flex: 1; min-width: 0; }
.dashboard-sandbox-diagnostics ul { list-style: none; margin: 7px 0 0; padding: 0; }
.dashboard-sandbox-diagnostics li { display: flex; justify-content: space-between; gap: 8px; padding: 4px 0; font-size: 11px; }
.dashboard-sandbox-diagnostics li + li { border-top: 1px solid var(--line); }
.dashboard-sandbox-diagnostics li button { padding: 0; border: 0; background: none; color: var(--accent); cursor: pointer; text-align: left; }
.dashboard-sandbox-diagnostics li span { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }

@media (max-width: 800px) {
.dashboard-sandbox-insight-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); }
.dashboard-sandbox-diagnostics { flex-direction: column; }
}

.dashboard-runtime-metric {
display: grid;
min-width: 0;
Expand Down
6 changes: 6 additions & 0 deletions apps/web/src/features/dashboard/DashboardView.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,12 @@ describe("Dashboard loaded-result presentation", () => {
expect(html).toContain("Cumulative CPU / capacity");
expect(html).toContain("1m 13s / 2 cores");
expect(html).toContain("512 MiB / 2.00 GiB");
expect(html).toContain("Sandbox diagnostics");
expect(html).toContain("Memory ≥80%");
const withoutMemory = render({
runtimeSnapshot: { sessions: [hosted], observations: [{ ...observation, memory: { usage_bytes: null, limit_bytes: null } }], loadedAt: 1_700_000_100_000 },
});
expect(withoutMemory).toMatch(/Memory ≥80%<\/small><strong>Unavailable<\/strong>/);
expect(html).toContain('aria-label="Runtime live-window charts"');
expect(html).toContain("Resource trends");
expect(html).toContain("Browser-local samples · reset on reload");
Expand Down
32 changes: 32 additions & 0 deletions apps/web/src/features/dashboard/RuntimeObservabilityContent.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import {

import {
buildRuntimeDashboardModel,
buildSandboxInsights,
formatDashboardBytes,
formatDashboardDuration,
formatDashboardTimestamp,
Expand Down Expand Up @@ -312,6 +313,9 @@ export function RuntimeObservabilityContent({
[snapshot, tokenTotals],
);
const summary = model.summary;
const sandbox = useMemo(() => buildSandboxInsights(model.rows), [model.rows]);
const unavailableReasons = Object.entries(sandbox.unavailableReasons)
.sort((left, right) => right[1] - left[1]);
return (
<>
<div className="dashboard-runtime-summary" aria-label={t("runtime.resourceSnapshot")}>
Expand All @@ -321,6 +325,34 @@ export function RuntimeObservabilityContent({
<RuntimeMetric icon={<Gauge size={17} />} label={t("runtime.metrics.tokens")} value={summary.totalTokens === null ? t("runtime.filters.unavailable") : formatDashboardTokens(summary.totalTokens, locale)} detail={t("runtime.metrics.tokenDetail", { covered: summary.tokenCoverageCount, total: summary.sessionCount })} />
</div>

<section className="dashboard-sandbox-insights" aria-labelledby="dashboard-sandbox-insights-title">
<header>
<div>
<h3 id="dashboard-sandbox-insights-title">{t("sandbox.title")}</h3>
<p>{t("sandbox.subtitle")}</p>
</div>
<small>{t(stale ? "sandbox.retained" : "sandbox.current")}</small>
</header>
<div className="dashboard-sandbox-insight-grid">
<div><small>{t("sandbox.observed")}</small><strong>{sandbox.observed.toLocaleString(locale)}</strong><span>{t("sandbox.observedDetail", { total: summary.managedRuntimeCount })}</span></div>
<div><small>{t("sandbox.unavailable")}</small><strong>{sandbox.unavailable.toLocaleString(locale)}</strong><span>{t("sandbox.unavailableDetail")}</span></div>
<div><small>{t("sandbox.highMemory")}</small><strong>{sandbox.measuredMemory === 0 ? t("runtime.filters.unavailable") : sandbox.highMemory.toLocaleString(locale)}</strong><span>{t("sandbox.highMemoryDetail", { total: sandbox.measuredMemory })}</span></div>
<div><small>{t("sandbox.parked")}</small><strong>{(sandbox.sleeping + sandbox.pending).toLocaleString(locale)}</strong><span>{t("sandbox.parkedDetail", { sleeping: sandbox.sleeping, pending: sandbox.pending })}</span></div>
</div>
{unavailableReasons.length > 0 || sandbox.highestMemory.length > 0 ? (
<div className="dashboard-sandbox-diagnostics">
<div>
<h4>{t("sandbox.sampleGaps")}</h4>
{unavailableReasons.length ? <ul>{unavailableReasons.map(([reason, count]) => <li key={reason}><span>{t(`runtime.status.${reason}` as never)}</span><strong>{count}</strong></li>)}</ul> : <p>{t("sandbox.noGaps")}</p>}
</div>
<div>
<h4>{t("sandbox.memoryLeaders")}</h4>
{sandbox.highestMemory.length ? <ul>{sandbox.highestMemory.map((entry) => <li key={entry.allocationId}><button type="button" onClick={() => onOpenSession(entry.sessionId)}>{entry.title}</button><span title={`${formatDashboardBytes(entry.memoryUsageBytes)} / ${formatDashboardBytes(entry.memoryLimitBytes)}`}>{entry.memoryPercent.toLocaleString(locale, { maximumFractionDigits: 1 })}%</span></li>)}</ul> : <p>{t("sandbox.noMemory")}</p>}
</div>
</div>
) : null}
</section>

<RuntimeTrendPanel snapshot={snapshot} stale={stale} loadRuntimeHistory={loadRuntimeHistory} />

<details className="dashboard-runtime-explorer">
Expand Down
20 changes: 10 additions & 10 deletions apps/web/src/features/dashboard/RuntimeTrendCharts.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ describe("Runtime live-window chart accessibility", () => {
})).toBe("Memory usage all series hidden; use the legend to show a series");
});

it("renders an unavailable current value as zero without retaining a stale value", () => {
it("renders an unavailable current value as a gap without retaining a stale value", () => {
const unavailable = {
...sample(120_000, null),
activeSandboxCount: 0,
Expand All @@ -58,13 +58,13 @@ describe("Runtime live-window chart accessibility", () => {
<RuntimeTrendCharts samples={[sample(60_000, .5), unavailable]} />,
);

expect(html).toContain("Runtime worker</th><td>0%</td><td>0</td>");
expect(html).toContain("Runtime worker</th><td>Unavailable</td><td>1</td>");
expect(html).not.toContain("Runtime worker</th><td>50%</td><td>1</td>");
expect(html).toContain("used</th><td>0 B</td><td>0</td>");
expect(html).toContain("used</th><td>Unavailable</td><td>1</td>");
expect(html).toContain("active</th><td>0</td><td>0</td>");
});

it("renders empty retained buckets as continuous zero-value chart series", () => {
it("renders empty retained buckets as missing measurements", () => {
const empty = (sampledAt: number): RuntimeTrendSample => ({
...sample(sampledAt, null),
targets: [],
Expand All @@ -78,13 +78,13 @@ describe("Runtime live-window chart accessibility", () => {
<RuntimeTrendCharts samples={[empty(60_000), empty(120_000)]} source="durable" />,
);

expect(html).toContain("usage</th><td>0%</td><td>0</td>");
expect(html).toContain("used</th><td>0 B</td><td>0</td>");
expect(html).toContain("usage</th><td>Unavailable</td><td>2</td>");
expect(html).toContain("used</th><td>Unavailable</td><td>2</td>");
expect(html).toContain("active</th><td>0</td><td>0</td>");
expect(html).toContain("input</th><td>0/min</td><td>0</td>");
expect(html).not.toContain("No retained CPU samples");
expect(html).not.toContain("No complete retained memory samples");
expect(html).not.toContain("No retained token samples");
expect(html).toContain("input</th><td>Unavailable</td><td>2</td>");
expect(html).toContain("No retained CPU samples");
expect(html).toContain("No retained observed memory samples");
expect(html).toContain("No retained token samples");
});

it("renders uPlot chart mounts and reports trends only after two real samples", () => {
Expand Down
19 changes: 11 additions & 8 deletions apps/web/src/features/dashboard/RuntimeTrendCharts.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -537,13 +537,16 @@ export function RuntimeTrendCharts({
id,
label: targetLabel(samples, id),
tone: tones[index] ?? "blue",
points: samples.map((sample) => ({ sampledAt: sample.sampledAt, value: (sample.targets.find((target) => target.seriesId === id)?.cpuRatio ?? 0) * 100 })),
points: samples.map((sample) => {
const ratio = sample.targets.find((target) => target.seriesId === id)?.cpuRatio;
return { sampledAt: sample.sampledAt, value: ratio == null ? null : ratio * 100 };
}),
}));
if (cpu.length === 0 && samples.length > 0) {
cpu.push({ id: "cpu", label: t("charts.usage"), tone: "orange", points: samples.map((sample) => ({ sampledAt: sample.sampledAt, value: 0 })) });
cpu.push({ id: "cpu", label: t("charts.usage"), tone: "orange", points: samples.map((sample) => ({ sampledAt: sample.sampledAt, value: null })) });
}
const memoryUsed = samples.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.memoryUsageBytes ?? 0 }));
const memoryLimit = samples.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.memoryLimitBytes ?? 0 }));
const memoryUsed = samples.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.memoryUsageBytes }));
const memoryLimit = samples.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.memoryLimitBytes }));
const active = [{
id: "active",
label: t(activeDisplay === "binary" ? "charts.runtime" : "charts.active.series"),
Expand All @@ -552,8 +555,8 @@ export function RuntimeTrendCharts({
points: samples.map((sample) => ({
sampledAt: sample.sampledAt,
value: activeDisplay === "binary"
? (sample.activeSandboxCount ?? 0) > 0 ? 1 : 0
: sample.activeSandboxCount ?? 0,
? sample.activeSandboxCount === null ? null : sample.activeSandboxCount > 0 ? 1 : 0
: sample.activeSandboxCount,
})),
}] satisfies TrendSeries[];
const throughput = tokenThroughput(samples);
Expand All @@ -565,8 +568,8 @@ export function RuntimeTrendCharts({
] satisfies TrendSeries[],
active,
tokens: [
{ id: "input", label: t("charts.input"), tone: "orange", points: throughput.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.inputPerMinute ?? 0 })) },
{ id: "output", label: t("charts.output"), tone: "green", points: throughput.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.outputPerMinute ?? 0 })) },
{ id: "input", label: t("charts.input"), tone: "orange", points: throughput.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.inputPerMinute })) },
{ id: "output", label: t("charts.output"), tone: "green", points: throughput.map((sample) => ({ sampledAt: sample.sampledAt, value: sample.outputPerMinute })) },
] satisfies TrendSeries[],
};
}, [activeDisplay, samples, t]);
Expand Down
51 changes: 51 additions & 0 deletions apps/web/src/features/dashboard/dashboard-model.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import type { AgentSession, RuntimeObservation, SavedAgent, TokenUsage } from "@
import {
buildDashboardSnapshot,
buildRuntimeDashboardModel,
buildSandboxInsights,
dashboardEnvironmentLabel,
dashboardEnvironmentProfile,
dashboardStatusLabel,
Expand Down Expand Up @@ -355,6 +356,56 @@ describe("Dashboard loaded-snapshot model", () => {
totalTokens: 26,
tokenCoverageCount: 2,
});
expect(buildSandboxInsights(model.rows)).toMatchObject({
observed: 1,
unavailable: 0,
highMemory: 0,
measuredMemory: 1,
highestMemory: [{ sessionId: second.id, memoryPercent: 37.5 }],
});
});

it("separates sleeping allocations from unexpected sampling gaps", () => {
const base = session("11111111-1111-4111-8111-111111111111");
const observation: RuntimeObservation = {
id: base.id, object: "agent.runtime_observation", session_id: base.id,
environment_id: "33333333-3333-4333-8333-333333333333", mode: "openai_hosted",
provider_type: "docker",
instance: { kind: "managed_allocation", allocation_id: "allocation-1", device_id: null, connection_generation: null },
lifecycle_state: "sleeping", status: "unavailable", reason: "runtime_not_running",
allocation_created_at: 100, resolved_at: 220, observed_at: null, started_at: null,
cpu: null, memory: null,
};
const failed = { ...observation, id: "other", session_id: "other", instance: { ...observation.instance, allocation_id: "allocation-2" }, lifecycle_state: "active" as const, reason: "sample_timeout" as const };
const rows = buildRuntimeDashboardModel([base, session("other")], [observation, failed]).rows;
expect(buildSandboxInsights(rows)).toMatchObject({
sleeping: 1, pending: 0, unavailable: 1,
unavailableReasons: { sample_timeout: 1 },
});
});

it("does not pick a conflicting same-second allocation observation by list order", () => {
const first = session("first");
const second = session("second");
const base: RuntimeObservation = {
id: first.id, object: "agent.runtime_observation", session_id: first.id,
environment_id: "33333333-3333-4333-8333-333333333333", mode: "openai_hosted",
provider_type: "docker", instance: { kind: "managed_allocation", allocation_id: "allocation-1", device_id: null, connection_generation: null },
lifecycle_state: "active", status: "observed", reason: null,
allocation_created_at: 100, resolved_at: 220, observed_at: 210, started_at: 150,
cpu: null, memory: { usage_bytes: 900, limit_bytes: 1000 },
};
const conflict: RuntimeObservation = {
...base, id: second.id, session_id: second.id, status: "unavailable", reason: "sample_timeout",
observed_at: null, started_at: null, cpu: null, memory: null,
};
const insights = (observations: RuntimeObservation[]) => buildSandboxInsights(buildRuntimeDashboardModel([first, second], observations).rows);
for (const order of [[base, conflict], [conflict, base]]) {
expect(insights(order)).toMatchObject({
observed: 0, unavailable: 1, highMemory: 0, measuredMemory: 0,
unavailableReasons: { sample_unavailable: 1 },
});
}
});

it("holds each Session's last reported tokens in the summary while public usage is null", () => {
Expand Down
Loading
Loading