diff --git a/apps/cockpit/scripts/manual-spec-freshness.spec.ts b/apps/cockpit/scripts/manual-spec-freshness.spec.ts new file mode 100644 index 000000000..df647cdec --- /dev/null +++ b/apps/cockpit/scripts/manual-spec-freshness.spec.ts @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT +// +// Tripwire for the `*.manual.ts` tier. +// +// Manual specs are live-LLM checks a human runs by hand: every cockpit +// playwright config matches `**/*.spec.ts`, so nothing executes these and CI +// can never notice when they drift. When this guard was written, 10 of 34 were +// asserting UI copy that existed nowhere in the repo — panel headings that had +// been renamed, and empty-state strings from sidebars that no longer exist. +// +// Running the specs themselves in CI isn't viable (real model, per-example dev +// server). What IS cheap is checking that everything they assert still exists +// in the source. That catches the rot this tier actually suffers — stale copy +// and dropped selectors — without booting anything. +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const REPO_ROOT = resolve(__dirname, '../../..'); +const SKIP_DIRS = new Set(['node_modules', 'dist', '.angular', '.venv', '.next', 'coverage']); + +function walk(dir: string, out: string[] = []): string[] { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return out; + } + for (const entry of entries) { + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) walk(join(dir, entry.name), out); + } else { + out.push(join(dir, entry.name)); + } + } + return out; +} + +const allFiles = walk(join(REPO_ROOT, 'cockpit')).concat(walk(join(REPO_ROOT, 'libs'))); +const manualSpecs = allFiles.filter((f) => f.endsWith('.manual.ts')); + +/** Every source file a manual spec could legitimately be asserting against. */ +const sourceCorpus = allFiles + .filter((f) => /\.(ts|html|css)$/.test(f) && !f.endsWith('.manual.ts')) + .map((f) => { + try { + return readFileSync(f, 'utf8'); + } catch { + return ''; + } + }) + .join('\n'); + +/** `text=Some copy` and `getByText('Some copy')` assertions. */ +function assertedText(spec: string): string[] { + const out = new Set(); + for (const m of spec.matchAll(/text=([^'"`)]{4,80})/g)) out.add(m[1].trim()); + for (const m of spec.matchAll(/getByText\(\s*['"]([^'"]{4,80})['"]/g)) out.add(m[1].trim()); + return [...out]; +} + +/** `page.locator('some-element')` custom-element selectors. */ +function assertedSelectors(spec: string): string[] { + const out = new Set(); + for (const m of spec.matchAll(/locator\(\s*['"]([a-z][a-z0-9]*(?:-[a-z0-9]+)+)['"]/g)) out.add(m[1]); + return [...out]; +} + +describe('manual e2e specs stay in sync with the source', () => { + it('finds the manual tier', () => { + // Guards the guard: if the glob ever stops matching, every assertion below + // would pass over an empty list and this file would be worthless. + expect(manualSpecs.length).toBeGreaterThan(20); + }); + + it.each(manualSpecs.map((f) => [f.slice(REPO_ROOT.length + 1), f]))( + '%s asserts only text that still exists', + (_label, file) => { + const spec = readFileSync(file, 'utf8'); + const missing = assertedText(spec).filter((t) => !sourceCorpus.includes(t)); + expect(missing, `copy asserted by this manual spec no longer exists anywhere in the repo`).toEqual([]); + }, + ); + + it.each(manualSpecs.map((f) => [f.slice(REPO_ROOT.length + 1), f]))( + '%s targets only selectors that still exist', + (_label, file) => { + const spec = readFileSync(file, 'utf8'); + const missing = assertedSelectors(spec).filter((s) => !sourceCorpus.includes(s)); + expect(missing, `element selectors targeted by this manual spec no longer exist`).toEqual([]); + }, + ); +}); diff --git a/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts b/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts index 6bbc1bad8..2e997ed6e 100644 --- a/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts +++ b/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts @@ -18,10 +18,10 @@ test.describe('AG-UI Interrupts Example', () => { await page.waitForSelector('app-interrupts', { state: 'attached' }); }); - test('renders the chat interface with approvals sidebar', async ({ page }) => { + test('renders the chat interface with refund suggestions', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=No pending approvals')).toBeVisible(); + await expect(page.locator('text=Refund a duplicate charge')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts b/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts index b300259c0..6d192e231 100644 --- a/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts +++ b/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Filesystem Example', () => { test('renders the chat interface with file operations sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask the agent to read or write a file.')).toBeVisible(); + await expect(page.locator('text=No file operations yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts b/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts index 8e2f6d1a2..21d64cec6 100644 --- a/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts +++ b/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Memory Example', () => { test('renders the chat interface with memory sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Tell the agent something about yourself to see it remember.')).toBeVisible(); + await expect(page.locator('text=No facts learned yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts b/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts index 608bd516f..e90e4bdc9 100644 --- a/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts +++ b/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Planning Example', () => { test('renders the chat interface with plan sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask a complex question to see the plan.')).toBeVisible(); + await expect(page.locator('text=No plan yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts b/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts index a0a70b7ec..f177b6e36 100644 --- a/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts +++ b/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Sandboxes Example', () => { test('renders the chat interface with execution log sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask the agent to write and run Python code.')).toBeVisible(); + await expect(page.locator('text=No code executed yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts b/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts index 48fe4f979..05fd2b405 100644 --- a/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts +++ b/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Skills Example', () => { test('renders the chat interface with skill invocation sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask the agent to calculate, count words, or summarize text.')).toBeVisible(); + await expect(page.locator('text=No skills invoked yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts b/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts index 772857d66..7cfa4d1e2 100644 --- a/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts +++ b/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Subagents Example', () => { test('renders the chat interface with subagents sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask a question to see subagent activity.')).toBeVisible(); + await expect(page.locator('text=No delegations yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts b/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts index 5c943f59e..e8b2cd8ec 100644 --- a/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts +++ b/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts @@ -9,7 +9,7 @@ test.describe('LangGraph Durable Execution Example', () => { test('renders the chat interface with execution status sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Execution Status')).toBeVisible(); + await expect(page.locator('text=Pipeline')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts b/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts index 7025339c5..847728aa6 100644 --- a/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts +++ b/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts @@ -6,10 +6,10 @@ test.describe('LangGraph Interrupts Example', () => { await page.waitForSelector('app-interrupts', { state: 'attached' }); }); - test('renders the chat interface with approvals sidebar', async ({ page }) => { + test('renders the chat interface with refund suggestions', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=No pending approvals')).toBeVisible(); + await expect(page.locator('text=Refund a duplicate charge')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts b/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts index 6b76ca5da..5222c4b99 100644 --- a/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts +++ b/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts @@ -9,7 +9,7 @@ test.describe('LangGraph Memory Example', () => { test('renders the chat interface with memory sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Agent Memory')).toBeVisible(); + await expect(page.locator('text=Learned Facts')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => {