diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 923d9a4d..21631f6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,6 +157,12 @@ jobs: - name: Load harness contract tests run: npm run test:load-harness + - name: POSIX API tests + run: npm run test:posix -w @chess-platform/api + + - name: POSIX load harness tests + run: npm run test:load-harness:posix + # The container images build with their own package chain, which CI never exercises — it # builds from the root chain instead. That gap let `docker compose up --build` stay broken # from M11 inc 5 onwards while every gate here was green. Static, so it costs seconds. @@ -199,6 +205,10 @@ jobs: # stopped looking at anything. These drive it against synthetic sources: a commented-out # entry, a forward migration, a constraint replaced rather than edited, a renamed # declaration. Pure functions over temp files, no services. + # Every test file in the repository must belong to an explicit, appropriately provisioned suite. + - name: Test topology check + run: npm run check:test-topology + - name: Guard script tests run: npm run test:scripts @@ -403,10 +413,13 @@ jobs: run: npm run build - name: Test persistence against Postgres - run: npm test --workspace @chess-platform/persistence + run: npm run test:integration:postgres --workspace @chess-platform/persistence - name: Test API concurrency controls against Postgres - run: npm test --workspace @chess-platform/api + run: npm run test:integration:postgres --workspace @chess-platform/api + + - name: Test scripts integration against Postgres + run: npm run test:scripts:integration # The only job that runs a real engine binary (ADR-0113). Every other analysis test drives a # fake transport or a provider double, which keeps the main suite hermetic but leaves the diff --git a/.github/workflows/live-provider.yml b/.github/workflows/live-provider.yml new file mode 100644 index 00000000..6e0d4207 --- /dev/null +++ b/.github/workflows/live-provider.yml @@ -0,0 +1,75 @@ +name: Live Provider Contract Tests + +# Dedicated, manual-only workflow for live third-party AI provider contract tests. +# These tests make real network calls to OpenAI / Anthropic APIs and require live API keys. +# Per repository policy: +# - They MUST NOT run in automatic PR CI. +# - They MUST NOT be marked as passed when credentials are unavailable. +# - They execute strictly on demand when credentials are provided in repository secrets. + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + live-provider-contract: + name: live provider contract tests (provisioned providers) + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Node 22.x + uses: actions/setup-node@v4 + with: + node-version: '22.x' + cache: npm + + - name: Install dependencies (reproducible) + run: npm ci + + - name: Build (dependency order) + run: npm run build + + - name: Detect provisioned credentials + id: credentials + run: | + has_openai=false + has_anthropic=false + if [ -n "${OPENAI_API_KEY}" ]; then has_openai=true; fi + if [ -n "${ANTHROPIC_API_KEY}" ]; then has_anthropic=true; fi + echo "has_openai=${has_openai}" >> "$GITHUB_OUTPUT" + echo "has_anthropic=${has_anthropic}" >> "$GITHUB_OUTPUT" + if [ "${has_openai}" = false ] && [ "${has_anthropic}" = false ]; then + echo "::error::At least one live-provider credential must be configured." + exit 1 + fi + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Test AI orchestrator OpenAI contract + if: steps.credentials.outputs.has_openai == 'true' + run: npm run test:live-provider:openai --workspace @chess-platform/ai-orchestrator + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Test AI orchestrator Anthropic contract + if: steps.credentials.outputs.has_anthropic == 'true' + run: npm run test:live-provider:anthropic --workspace @chess-platform/ai-orchestrator + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + + - name: Test AI features OpenAI contract + if: steps.credentials.outputs.has_openai == 'true' + run: npm run test:live-provider:openai --workspace @chess-platform/ai-features + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + + - name: Test AI features Anthropic contract + if: steps.credentials.outputs.has_anthropic == 'true' + run: npm run test:live-provider:anthropic --workspace @chess-platform/ai-features + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} diff --git a/deploy/load/test/run-evidence.posix.test.mjs b/deploy/load/test/run-evidence.posix.test.mjs new file mode 100644 index 00000000..3c107863 --- /dev/null +++ b/deploy/load/test/run-evidence.posix.test.mjs @@ -0,0 +1,46 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..'); + +const armScript = (dir, body, options = 'undefined') => ` +import { existsSync, writeFileSync } from 'node:fs'; +import { armFailureEvidence, buildEvidence, clearEvidence, writeEvidence } from ${JSON.stringify( + pathToFileURL(join(REPO_ROOT, 'scripts/lib/run-evidence.mjs')).href, +)}; +const DIR = ${JSON.stringify(dir)}; +const FILE = 'evidence.json'; +const build = (exitCode) => buildEvidence({ + harness: 'test', outcome: 'aborted', exitCode, + startedAt: '2026-08-27T10:00:00.000Z', finishedAt: '2026-08-27T10:00:01.000Z', +}); +clearEvidence(DIR, FILE); +const fallback = armFailureEvidence(DIR, FILE, build, ${options}); +${body} +`; + +test('a real SIGTERM leaves the artifact and still terminates by the signal', async () => { + const dir = mkdtempSync(join(tmpdir(), 'gambit-evidence-')); + const file = join(dir, 'child.mjs'); + writeFileSync(file, armScript(dir, "console.log('armed');\nsetTimeout(() => {}, 60_000);"), 'utf8'); + + const child = spawn(process.execPath, [file], { cwd: REPO_ROOT, encoding: 'utf8' }); + await once(child.stdout, 'data'); + child.kill('SIGTERM'); + const [code, signal] = await once(child, 'exit'); + + assert.equal( + signal, + 'SIGTERM', + 'the handler re-raises after writing, so the process still dies BY the signal rather than ' + + 'turning an interrupt into an ordinary exit', + ); + assert.equal(code, null); + assert.equal(JSON.parse(readFileSync(join(dir, 'evidence.json'), 'utf8')).exitCode, 143); +}); diff --git a/deploy/load/test/run-evidence.test.mjs b/deploy/load/test/run-evidence.test.mjs index 2b3294cf..0cfc4128 100644 --- a/deploy/load/test/run-evidence.test.mjs +++ b/deploy/load/test/run-evidence.test.mjs @@ -413,30 +413,6 @@ test('an interrupt after clearing still leaves a failure artifact', () => { assert.notEqual(result.status, 0, 'and an interrupted run must never look like a successful one'); }); -test( - 'a real SIGTERM leaves the artifact and still terminates by the signal', - { skip: process.platform === 'win32' ? 'Windows terminates on kill() without running handlers' : false }, - async () => { - const dir = mkdtempSync(join(tmpdir(), 'gambit-evidence-')); - const file = join(dir, 'child.mjs'); - writeFileSync(file, armScript(dir, "console.log('armed');\nsetTimeout(() => {}, 60_000);"), 'utf8'); - - const child = spawn(process.execPath, [file], { cwd: REPO_ROOT, encoding: 'utf8' }); - await once(child.stdout, 'data'); - child.kill('SIGTERM'); - const [code, signal] = await once(child, 'exit'); - - assert.equal( - signal, - 'SIGTERM', - 'the handler re-raises after writing, so the process still dies BY the signal rather than ' + - 'turning an interrupt into an ordinary exit', - ); - assert.equal(code, null); - assert.equal(JSON.parse(readFileSync(join(dir, 'evidence.json'), 'utf8')).exitCode, 143); - }, -); - test('a run that wrote its own evidence is not overwritten by the fallback', () => { const dir = mkdtempSync(join(tmpdir(), 'gambit-evidence-')); const result = runInChild( diff --git a/docs/PROJECT_STATE.md b/docs/PROJECT_STATE.md index d453abac..c1e4395f 100644 --- a/docs/PROJECT_STATE.md +++ b/docs/PROJECT_STATE.md @@ -6,7 +6,19 @@ > to read **only this file** and continue immediately. Updated after every > milestone and every significant architectural step. -_Last updated: 2026-09-22 — M15 Increment 60: hash-aware immutable asset delivery contract._ +_Last updated: 2026-09-22 — M15 Increment 61: exact-accounting zero-skip correction and PR #56 integration._ + +Prior: _Last updated: 2026-09-22 — M15 Increment 60: hash-aware immutable asset delivery contract._ + +Prior: _Last updated: 2026-09-21 — M15 Increment 59h: Authoritative Playwright CLI test reachability discovery and fail-closed validation._ + +Prior: _Last updated: 2026-09-17 — M15 Increment 59g: Negative lookbehind directive isolation, unnumbered TAP directive support, and zero-plan fail closed._ + +Prior: _Last updated: 2026-09-17 — M15 Increment 59f: Shared test-output parser module and strict TAP directive/not-ok reconciliation._ + +Prior: _Last updated: 2026-09-17 — M15 Increment 59d: Centralized repository-wide hermetic zero-skip orchestration and live-provider contract alignment._ + +Prior: _Last updated: 2026-09-17 — M15 Increment 59c: Zero test-skip CI architecture — anchored reporter summary parsing and decoy output hardening._ Prior: _Last updated: 2026-09-16 — M15 Increment 59: production web delivery caching and compression contract._ @@ -4240,6 +4252,82 @@ Per package: `cd packages/ && npm install && npm run build && npm test`. - Reserve backup files exclusively and clean up only resources created by the drill. Preserve restore and cleanup errors together while continuing other cleanup. Redact connection secrets from diagnostics, including CLI argument errors. - Verify append-only protection and valid, ready HNSW indexes on their specific public-schema relations. Added database-boundary and disposable-file regressions for native/Docker custom/plain orchestration and failure paths; live integration remains opt-in and was not run against an existing database. +## M15 Increment 59 — Zero test-skip CI architecture and suite partitioning — 2026-09-17 + +- **Problem solved**: The repository previously allowed test self-skipping across packages when services (PostgreSQL, Stockfish, live AI keys) were not provisioned. In PR CI, dozens of tests were skipped in `build-test` (`packages/persistence`, `packages/api`, `packages/ai-orchestrator`, `packages/ai-features`, `scripts/test`). This violated the owner's strict zero-test-skip gate (`failed = 0`, `skipped = 0`). +- **Partitioned suite architecture**: + - Hermetic Unit Suites (`build-test`): `npm test` across all workspaces runs purely hermetic tests with zero services, zero network, zero skips. + - PostgreSQL Integration Suites (`postgres-integration`): `npm run test:integration:postgres` for `persistence` and `api`, and `npm run test:scripts:integration` run against real PostgreSQL 16 + pgvector (`DATABASE_URL`) with zero skips. + - Engine Smoke Suite (`analysis-smoke`): `npm run test:analysis-smoke -w @chess-platform/api` runs against pinned Stockfish 16, Fairy-Stockfish 14, and real PostgreSQL with zero skips. + - Gateway Service Suite (`gateway-service`): runs against real Redis 7 and Nginx with zero skips. + - Acceptance Suite (`m6-acceptance`): runs Playwright Chromium e2e tests with zero skips. + - Dedicated Live Provider Workflow (`.github/workflows/live-provider.yml`): third-party AI provider contract tests (OpenAI / Anthropic) extracted into dedicated files and run strictly via `workflow_dispatch` with verified secrets. They never run in PR CI and never produce misleading skipped counts. +- **Enforcement & Invariant Guards**: + - `scripts/run-zero-skip.mjs`: Programmatic test runner wrapper that monitors TAP/spec output and fails if `skipped > 0`. + - `scripts/check-test-topology.mjs`: Invariant guard verifying that every test file in the monorepo is classified into an explicit suite and none are orphaned. + - `scripts/test-counts.mjs`: Detailed test suite count and skip auditor reporting per-suite pass/fail/skip totals. +- Detailed in `docs/adr/0142-zero-test-skip-ci-architecture.md`. + +## M15 Increment 59a — Zero test-skip CI architecture: ChatGPT blocker fixes and PR overlap disclosure — 2026-09-17 + +Addresses four blocking review findings identified by ChatGPT independent review on PR #57 (`gemini/ci-zero-skip-architecture`, HEAD `27c3c66`): + +- **Blocker 1 — Live provider self-contained build**: `test:live-provider` in `packages/ai-orchestrator` and `packages/ai-features` now unconditionally prepends `npm run build:test &&`. Previously the script ran `dist-test` files without guaranteeing compilation, causing spurious failures when invoked in isolation. +- **Blocker 2 — False-green on missing test output**: `scripts/run-zero-skip.mjs` now requires `totalTests !== null && totalTests > 0`. A process that exits 0 with arbitrary text and no test-count line fails with "No executed tests detected". Regex updated to recognise TAP plan (`1..N`) and `tests: N` formats. Two regression tests added to `scripts/test/zero-skip-enforcement.test.mjs`. +- **Blocker 3 — POSIX suite partition and Windows laundering removal**: Three POSIX-only tests extracted into dedicated `*.posix.test.ts` / `*.posix.test.mjs` files. Dedicated `api-posix-unit` and `load-harness-posix` CI steps added (Linux only). The entire `if (process.platform === 'win32')` skip-laundering block removed from `scripts/run-zero-skip.mjs`. Topology passes: 396 files, 20 suites, 0 unclassified. +- **Blocker 4 — PR #56 overlap disclosure**: PR #57 shares 4 files with open PR #56 (`gemini/web-delivery-cache-compression`): `.github/workflows/ci.yml`, `docs/PROJECT_STATE.md`, `scripts/ci-local.mjs`, `services/gateway/package.json`. PR #57 is foundational; PR #56 must be rebased after PR #57 lands. PR #55 has zero file overlap with PR #57. Overlap table documented in `docs/adr/0142-zero-test-skip-ci-architecture.md` §"Open PR Overlap". + +## M15 Increment 59b — PostgreSQL backup drill teardown resilience and skip removal — 2026-09-17 + +- **Backup-restore drill connection resilience**: Attached error handlers and tracked client sockets on `sourcePool`, `targetPool`, and `adminClient` in `scripts/db-backup-restore-drill.mjs`. When `DROP DATABASE ... WITH (FORCE)` terminates idle target pool connections, the resulting `FATAL: terminating connection due to administrator command` (57P01) or socket termination is absorbed rather than bubbling up as an `uncaughtException` in `node:test`. +- **Zero-skip compliance**: Removed the lingering `skip: process.env.DATABASE_URL ? false : ...` condition from `scripts/test/backup-restore-drill.integration.test.mjs`, enforcing `DATABASE_URL` presence via assertion so the test never skips under any environment. + +## M15 Increment 59c — Zero test-skip parser anchoring and decoy output hardening — 2026-09-17 + +- **Anchored skip count parsing**: In `scripts/run-zero-skip.mjs`, anchored the skip summary regex to genuine line-anchored reporter summary lines (`/^\s*(?:#|ℹ)\s+skipped:?\s+(\d+)\b/gim`). Prose outputs containing words like "skipped 1" (e.g. application logs) no longer trigger false-positive suite failures when the reporter summary reports 0 skips. +- **Anchored test count parsing**: Anchored the test count regex strictly to reporter summary lines (`/^\s*(?:#|ℹ)\s+tests:?\s+(\d+)\b/gim`) and line-anchored TAP plan headers (`/^\s*1\.\.(\d+)\b/gm`). Arbitrary prose like "unrelated tests 0" or "we ran tests 5" can no longer cause false failures or false greens. +- **Regression coverage**: Added 8 comprehensive regression tests in `scripts/test/zero-skip-enforcement.test.mjs` verifying that decoy prose ("skipped N", "tests N", unanchored "1..N") does not corrupt detection, and that genuine skips and missing test summaries are strictly enforced. +- **Documentation coverage**: Added JSDoc docstrings across `scripts/run-zero-skip.mjs`, `scripts/check-test-topology.mjs`, and `scripts/test-counts.mjs` to satisfy CodeRabbit maintainability standards. +- **Multi-summary aggregation**: Handled multi-suite test outputs in `scripts/run-zero-skip.mjs` by accumulating skips across all summary lines (preventing earlier skips from being laundered by later zero-skip suites) and requiring that every reported suite executed > 0 tests. + +## M15 Increment 59d — Centralized repository-wide hermetic zero-skip orchestration and live-provider contract alignment — 2026-09-17 + +- **Centralized hermetic zero-skip orchestrator (`scripts/run-hermetic-tests.mjs`)**: Established a centralized test runner executing all 19 hermetic workspace packages in sequence under programmatic `runWithZeroSkip` enforcement. Root `npm test` now routes directly to `node scripts/run-hermetic-tests.mjs`. Individual package test commands remain intact and untouched (preserving PR #55 boundaries). If any child workspace reports `skipped > 0`, zero executed tests, non-zero exit code, or signal termination, the orchestrator halts immediately and fails the run. +- **Cross-platform programmatic invocation**: The orchestrator resolves `process.env.npm_execpath` and invokes `process.execPath` directly with `shell: false`, bypassing shell-specific `$npm_execpath`/`%npm_execpath%` expansion issues and Node CVE-2024-27980 Windows `.cmd` spawn restrictions. +- **Live-provider contract alignment in `scripts/test-counts.mjs`**: Replaced permissive `OPENAI_API_KEY || ANTHROPIC_API_KEY` with strict conjunctions matching the actual runner contract: `OPENAI_API_KEY && ANTHROPIC_API_KEY` for `ai-orchestrator`, and `OPENAI_API_KEY && ANTHROPIC_API_KEY && GAMBIT_TEST_INTEGRATION=1` for `ai-features`. Incompletely provisioned suites are reported as `NOT EXECUTED` rather than invoked to self-skip. +- **Regression coverage**: Added 5 new regression tests in `scripts/test/zero-skip-enforcement.test.mjs` (24 tests total, all passing) demonstrating that child workspaces exiting 0 but reporting `# tests 1 \n # skipped 1` fail the repository-level runner, multi-workspace runs with multiple reporter summaries succeed when all report tests > 0 and skipped = 0, zero-test workspaces fail, and natural non-zero exit codes propagate. + +## M15 Increment 59e — Zero-skip enforcer and audit rejection of TODO and cancelled test metrics — 2026-09-17 + +- **TODO test rejection in `scripts/run-zero-skip.mjs`**: Enforced that `todo === 0` across all genuine line-anchored reporter summaries (`# todo N`, `ℹ todo N`) and individual test directives (`# TODO`). A Node test marked TODO is non-failing by default in Node.js, which previously allowed a suite with `pass = 0, todo = 1` to pass the quality gate without genuine passing assertions. The enforcer now strictly rejects `todo > 0` with exit code 1. +- **Cancelled test rejection**: Added explicit rejection for `cancelled > 0` across reporter summaries (`# cancelled N`, `ℹ cancelled N`) to prevent cancelled test runs from passing CI. +- **Audited test counts alignment in `scripts/test-counts.mjs`**: Updated `test-counts.mjs` to parse `pass`, `fail`, `skipped`, `todo`, and `cancelled` metrics across both TAP and spec reporter formats. Executed suites fail the audit if `failed > 0`, `skipped > 0`, `todo > 0`, `cancelled > 0`, or `tests === 0`. Output clearly separates `tests` from `passed` per suite (`tests X, passed Y, failed 0, skipped 0, todo 0, cancelled 0`) and grand total (`Grand Total Executed: 3636 tests (3636 passed, 0 failed, 0 skipped, 0 todo, 0 cancelled)`). +- **Regression coverage**: Added 10 new regression tests in `scripts/test/zero-skip-enforcement.test.mjs` (34 tests total, all passing) covering TODO-only output rejection, mixed pass + TODO rejection, clean summary acceptance, decoy prose ignoring, cancelled count rejection, spec format TODO/cancelled rejection, individual `# TODO` directive rejection, and multi-summary TODO/cancelled rejection. + +## M15 Increment 59f — Shared test-output parser module and strict TAP directive/not-ok reconciliation — 2026-09-17 + +- **Shared pure parser module (`scripts/lib/test-output-parser.mjs`)**: Centralized test metric, plan, and directive parsing across `scripts/run-zero-skip.mjs`, `scripts/test-counts.mjs`, and regression test suites. Fully annotated with complete JSDoc docstrings for CodeRabbit maintainability standards. +- **Summary and directive independent evaluation**: Fixed the flaw where `# skipped 0` or `# todo 0` prevented evaluation of record-level `# SKIP` or `# TODO` directives. `parseSkippedCount` and `parseTodoCount` independently evaluate summaries and individual directives, guaranteeing that any genuine record-level skip or TODO results in at least count 1 and triggers gate failure. +- **Line-anchored raw TAP failure detection (`RAW_TAP_FAILURE_REGEX`)**: Added detection for raw TAP `not ok` records lacking reporter summary lines when child processes exit 0. Preserves TAP semantics by properly isolating `# TODO` and `# SKIP` directives so they are not misclassified as ordinary failures while still failing their respective quality gates. +- **Per-summary zero-test preservation**: `parseTestCount` fails closed (returns 0) if ANY recognized summary or plan header reports 0 executed tests, preventing multi-summary runs (such as `# tests 0 \n # tests 5`) from laundering unexecuted test suites. +- **Auditor alignment in `scripts/test-counts.mjs`**: Replaced disparate ad-hoc regexes with shared parser helpers across both hermetic and service suite audit loops, enforcing identical zero-skip, zero-todo, and failure contracts across both execution and reporting paths. +- **Regression coverage**: Added 14 new tests in `scripts/test/zero-skip-enforcement.test.mjs` (48 tests total, all passing) verifying summary + directive reconciliation, raw TAP not-ok detection, multi-summary zero-test preservation, decoy prose immunity, and parser helper unit contracts. + +## M15 Increment 59g — Negative lookbehind directive isolation, unnumbered TAP directive support, and zero-plan fail closed — 2026-09-17 + +- **Negative lookbehind directive isolation (`(? && npm install && npm run build && npm test`. - Restricted one-year immutable caching to Vite-style content-hashed filenames under `/assets/`; existing unhashed assets now use `Cache-Control: no-cache`, and both hashed and unhashed missing assets remain strict 404 responses without immutable headers. - Extended the real-Nginx acceptance suite with a deterministic unhashed `/assets/runtime-config.json` fixture, hashed JS and CSS cache assertions, root static-file revalidation checks, both hashed/unhashed 404 paths, and shared security-header assertions while preserving gzip, SPA, REST, and WebSocket coverage. + +## M15 Increment 61 — Exact-accounting zero-skip correction and PR #56 integration (2026-09-22) + +- Integrated merged PR #56/current `main` into PR #57 with a history-preserving merge and retained its hash-aware immutable caching, gzip, WebSocket, CI trigger, and real-Nginx acceptance behavior. Classified `scripts/nginx-web-delivery-acceptance.mjs` as the twenty-first explicit suite; topology now verifies 397 test files across 21 suites. +- Added a Playwright reporter that fails an executed browser suite on zero tests, skipped outcomes, unexpected outcomes, or interrupted execution while leaving `--list` discovery non-executing. The `m6-acceptance` job therefore enforces zero skips on actual Playwright outcomes rather than trusting a green process status. +- Split live OpenAI and Anthropic contracts into independently selectable scripts and workflow steps. One provisioned credential now runs its full provider surface without registering the absent provider as skipped; both credentials run both surfaces, and no credentials fail closed. +- Hardened TAP/spec processing with ANSI removal, complete six-field summary accounting, complete plan-only TAP validation, pass/total reconciliation, malformed and contradictory transcript rejection, UTF-8-safe independent stdout/stderr streaming, and child signal forwarding with listener cleanup. `test-counts.mjs` consumes the same exact accounting and no longer synthesizes pass totals. +- Scoped deployment exclusions to the exact `deploy/helm` and `deploy/observability` trees, added portable negative-extglob fallback coverage, and added falsification regressions for package directories with colliding names, partial summaries, contradictory summaries, truncated plans, ANSI output, provider credential matrices, and Playwright skipped outcomes. diff --git a/docs/adr/0142-zero-test-skip-ci-architecture.md b/docs/adr/0142-zero-test-skip-ci-architecture.md new file mode 100644 index 00000000..4bc0bb8f --- /dev/null +++ b/docs/adr/0142-zero-test-skip-ci-architecture.md @@ -0,0 +1,81 @@ +# ADR-0142: Zero Test-Skip CI Architecture and Environment-Gated Suite Partitioning + +## Context + +The repository previously relied on self-skipping guards (`const skip = ...`) across various packages when external services or API credentials were not provisioned. This produced dozens of skipped tests during PR CI runs: +- `packages/persistence` skipped 21 PostgreSQL integration test files (48 tests) during hermetic `npm test` in the `build-test` job. +- `packages/api` skipped 7 PostgreSQL integration test files (39 tests) and 3 engine smoke test files when run without `DATABASE_URL` or engine binaries. +- `packages/ai-orchestrator` had 2 live completion tests embedded in `test/adapters.test.ts` that skipped without `OPENAI_API_KEY` or `ANTHROPIC_API_KEY`. +- `packages/ai-features` had 16 live provider integration tests across 9 files that skipped without API keys. +- `scripts/test/backup-restore-drill.test.mjs` contained a live database restore drill test that skipped when `DATABASE_URL` was not set. + +This pattern violated the owner's strict quality gate: **for every test suite executed in CI, `failed = 0` and `skipped = 0`**. A test must only be invoked in a suite that provides its required execution environment. + +## Decision + +We establish an explicit, partitioned test architecture across all packages, guarded by programmatic zero-skip enforcement and static topology validation: + +1. **Hermetic Unit Test Suites (`build-test` CI job)**: + - Run via `npm test` across all workspaces and `npm run test:scripts`. + - Purely hermetic: zero external network, zero external databases, zero live engine binaries. + - Every single executed test must pass with `skipped = 0`. + +2. **PostgreSQL Integration Test Suites (`postgres-integration` CI job)**: + - Run via `npm run test:integration:postgres --workspace @chess-platform/persistence`, `npm run test:integration:postgres --workspace @chess-platform/api`, and `npm run test:scripts:integration`. + - Requires real PostgreSQL with `vector` extension (`DATABASE_URL`). + - Genuinely executes all database persistence, concurrency, and backup drill tests with `skipped = 0`. + +3. **Engine Smoke Test Suite (`analysis-smoke` CI job)**: + - Run via `npm run test:analysis-smoke --workspace @chess-platform/api`. + - Requires pinned Stockfish 16, Fairy-Stockfish 14, and real PostgreSQL. + - Genuinely executes production composition tests with `skipped = 0`. + +4. **Gateway Service Suites (`gateway-service` CI job)**: + - Run via `npm test`, `npm run test:trusted-edge`, and `npm run test:web-delivery` in `services/gateway`. + - Requires real Redis 7 (`REDIS_URL`) and Docker/Nginx (`REQUIRE_DOCKER=1`). + - Genuinely executes command routing, lease ownership, edge proxy, and production cache/compression tests with `skipped = 0`. + +5. **M6 Acceptance Suite (`m6-acceptance` CI job)**: + - Run via `npm run e2e` in `packages/web`. + - Requires Playwright Chromium and the in-memory backend harness. + - Genuinely executes end-to-end user journeys with `skipped = 0`. + - `scripts/playwright-zero-skip-reporter.mjs` evaluates every discovered `TestCase.outcome()` and overrides an otherwise-green Playwright result when the suite is empty, skipped, interrupted, or has an unexpected result. Discovery-only `--list` commands remain read-only and do not apply execution policy. + +6. **Dedicated Live Provider Workflow (`.github/workflows/live-provider.yml`)**: + - Third-party live OpenAI and Anthropic contract tests are separated into `test:live-provider` scripts. + - Extracted live tests from `packages/ai-orchestrator/test/adapters.test.ts` into `packages/ai-orchestrator/test/adapters-live.integration.test.ts`. + - Extracted live backup restore test from `scripts/test/backup-restore-drill.test.mjs` into `scripts/test/backup-restore-drill.integration.test.mjs`. + - Live tests run strictly on demand via `workflow_dispatch`. OpenAI and Anthropic are selected and executed independently, so either single credential runs its complete provider contract; both credentials run both contracts, and no credentials fail the workflow. Test registration is controlled by `GAMBIT_LIVE_PROVIDER` without `skip` annotations. + - They do not run in PR CI and are never marked as passed without at least one selected credential. + +7. **Zero-Skip Enforcer (`scripts/run-zero-skip.mjs`)**: + - Wraps test invocations, streams runner output in real time, parses TAP/spec skip counts, and fails with exit code 1 if any test is skipped. + - Requires `totalTests > 0`: exits 1 with "No executed tests detected" when a process exits 0 with arbitrary text and no test summary, preventing false-green on misconfigured commands. + - Windows platform laundering removed: no skip bypass based on `process.platform`. + - ANSI control sequences are removed before parsing. Complete Node summaries must contain consistent tests/pass/fail/cancelled/skipped/todo accounting; plan-only TAP must contain one complete top-level plan with the matching number of test points. Partial, malformed, truncated, or contradictory output fails closed. + - Parent termination signals are forwarded to the child process and temporary signal handlers are removed on every exit path. + +8. **Topology Invariant Guard (`scripts/check-test-topology.mjs`)**: + - Scans all 397 test files across 21 explicit suites to verify that every test file is mapped and no file is orphaned or unclassified, including `scripts/nginx-web-delivery-acceptance.mjs` imported from merged PR #56. + - Enforced in `npm run check:test-topology` in `build-test` CI and `scripts/ci-local.mjs`. + - Deployment-only exclusions are path-scoped to `deploy/helm` and `deploy/observability`; identically named directories under packages cannot hide test files. The Node fallback matcher supports globstar and the negative extglob used by package scripts. + +9. **POSIX Suite Partition**: + - Tests that require POSIX-only OS guarantees (SIGTERM process-tree teardown, `chmod`/permission bits) are extracted from cross-platform files into dedicated `*.posix.test.ts` / `*.posix.test.mjs` files. + - Affected files: `packages/api/test/diagnostics/signature-b-correlate.posix.test.ts` (2 tests) and `deploy/load/test/run-evidence.posix.test.mjs` (1 test). + - These suites run in dedicated `api-posix-unit` and `load-harness-posix` CI steps (Linux only) so they genuinely execute with `skipped = 0` on CI and are excluded from Windows runs without laundering. + +10. **Live Provider Self-Contained Build**: + - `test:live-provider` in `packages/ai-orchestrator` and `packages/ai-features` now unconditionally prepends `npm run build:test &&`, ensuring live integration test files are compiled before the runner is invoked. + +## PR #56 Integration + +PR #56 (`perf(web): harden production caching and compression`) was merged to `main` before this correction. PR #57 merges that exact mainline state without rewriting its published history. The imported Nginx configuration, hash-aware cache contract, gzip acceptance assertions, `packages/web` delivery trigger, and local/hosted gateway job parity are preserved. Its new `scripts/nginx-web-delivery-acceptance.mjs` file is now an explicit topology and test-count suite rather than an unclassified exception. + +## Consequences + +- Zero tests are skipped in PR CI: every executed test genuinely runs and asserts its specification against its required environment. +- Flaky or unconfigured external services cannot cause false-green skipped test reports. +- Developers and reviewers get immediate notification if a new test is unclassified or improperly self-skipped. +- POSIX-only tests execute genuinely on Linux CI without any skip laundering; they are excluded on Windows without polluting hermetic suite counts. +- False-green on arbitrary text output is eliminated by requiring a positive test count before accepting an exit-0 result. diff --git a/package.json b/package.json index 959439ec..88963adf 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ ], "scripts": { "build": "npm run build --workspace @chess-platform/core && npm run build --workspace @chess-platform/search && npm run build --workspace @chess-platform/social && npm run build --workspace @chess-platform/messaging && npm run build --workspace @chess-platform/community && npm run build --workspace @chess-platform/achievements && npm run build --workspace @chess-platform/studies && npm run build --workspace @chess-platform/learning && npm run build --workspace @chess-platform/game && npm run build --workspace @chess-platform/tournament && npm run build --workspace @chess-platform/realtime-gateway && npm run build --workspace @chess-platform/engine && npm run build --workspace @chess-platform/anti-cheat && npm run build --workspace @chess-platform/persistence && npm run build --workspace @chess-platform/ai-orchestrator && npm run build --workspace @chess-platform/ai-features && npm run build --workspace @chess-platform/api && npm run build --workspace @chess-platform/web && npm run build --workspace @chess-platform/e2e-harness", - "test": "npm run test --workspace @chess-platform/core && npm run test --workspace @chess-platform/search && npm run test --workspace @chess-platform/social && npm run test --workspace @chess-platform/messaging && npm run test --workspace @chess-platform/community && npm run test --workspace @chess-platform/achievements && npm run test --workspace @chess-platform/studies && npm run test --workspace @chess-platform/learning && npm run test --workspace @chess-platform/game && npm run test --workspace @chess-platform/tournament && npm run test --workspace @chess-platform/realtime-gateway && npm run test --workspace @chess-platform/engine && npm run test --workspace @chess-platform/anti-cheat && npm run test --workspace @chess-platform/persistence && npm run test --workspace @chess-platform/api && npm run test --workspace @chess-platform/web && npm run test --workspace @chess-platform/e2e-harness && npm run test --workspace @chess-platform/ai-orchestrator && npm run test --workspace @chess-platform/ai-features", + "test": "node scripts/run-hermetic-tests.mjs", "lint": "npm run lint --workspace @chess-platform/core && npm run lint --workspace @chess-platform/search && npm run lint --workspace @chess-platform/social && npm run lint --workspace @chess-platform/messaging && npm run lint --workspace @chess-platform/community && npm run lint --workspace @chess-platform/achievements && npm run lint --workspace @chess-platform/studies && npm run lint --workspace @chess-platform/learning && npm run lint --workspace @chess-platform/game && npm run lint --workspace @chess-platform/tournament && npm run lint --workspace @chess-platform/realtime-gateway && npm run lint --workspace @chess-platform/engine && npm run lint --workspace @chess-platform/anti-cheat && npm run lint --workspace @chess-platform/persistence && npm run lint --workspace @chess-platform/api && npm run lint --workspace @chess-platform/web && npm run lint --workspace @chess-platform/e2e-harness && npm run lint --workspace @chess-platform/ai-orchestrator && npm run lint --workspace @chess-platform/ai-features", "clean": "npm run clean --workspace @chess-platform/core && npm run clean --workspace @chess-platform/search && npm run clean --workspace @chess-platform/social && npm run clean --workspace @chess-platform/messaging && npm run clean --workspace @chess-platform/community && npm run clean --workspace @chess-platform/achievements && npm run clean --workspace @chess-platform/studies && npm run clean --workspace @chess-platform/learning && npm run clean --workspace @chess-platform/anti-cheat && npm run clean --workspace @chess-platform/game && npm run clean --workspace @chess-platform/tournament && npm run clean --workspace @chess-platform/realtime-gateway && npm run clean --workspace @chess-platform/persistence && npm run clean --workspace @chess-platform/api && npm run clean --workspace @chess-platform/engine && npm run clean --workspace @chess-platform/web && npm run clean --workspace @chess-platform/e2e-harness && npm run clean --workspace @chess-platform/ai-orchestrator && npm run clean --workspace @chess-platform/ai-features", "build:server": "npm run build --workspace @chess-platform/core && npm run build --workspace @chess-platform/search && npm run build --workspace @chess-platform/social && npm run build --workspace @chess-platform/messaging && npm run build --workspace @chess-platform/community && npm run build --workspace @chess-platform/achievements && npm run build --workspace @chess-platform/studies && npm run build --workspace @chess-platform/learning && npm run build --workspace @chess-platform/game && npm run build --workspace @chess-platform/tournament && npm run build --workspace @chess-platform/realtime-gateway && npm run build --workspace @chess-platform/engine && npm run build --workspace @chess-platform/anti-cheat && npm run build --workspace @chess-platform/persistence && npm run build --workspace @chess-platform/ai-orchestrator && npm run build --workspace @chess-platform/ai-features && npm run build --workspace @chess-platform/api", @@ -20,13 +20,16 @@ "check:deploy-gates": "node scripts/check-deploy-gates.mjs", "load-test": "node scripts/load-test.mjs", "load-test:ws": "node scripts/ws-load-test.mjs", - "test:load-harness": "node --test \"deploy/load/test/**/*.test.mjs\"", + "test:load-harness": "node scripts/run-zero-skip.mjs -- node --test \"deploy/load/test/**/!(*.posix).test.mjs\"", + "test:load-harness:posix": "node scripts/run-zero-skip.mjs -- node --test \"deploy/load/test/**/*.posix.test.mjs\"", "test:gateway-redis": "node -e \"if(!process.env.REDIS_URL){console.error('REDIS_URL is required for test:gateway-redis (the Redis integration suite skips without it)');process.exit(1)}\" && npm run build --prefix services/gateway && npm test --prefix services/gateway", "check:adr-claims": "node scripts/check-adr-claims.mjs", "check:ci-parity": "node scripts/check-ci-parity.mjs", "check:variant-parity": "node scripts/check-variant-parity.mjs", "check:engine-pin-parity": "node scripts/check-engine-pin-parity.mjs", - "test:scripts": "node --test \"scripts/test/**/*.test.mjs\"", + "check:test-topology": "node scripts/check-test-topology.mjs", + "test:scripts": "node scripts/run-zero-skip.mjs -- node --test \"scripts/test/**/!(*.integration).test.mjs\"", + "test:scripts:integration": "node scripts/run-zero-skip.mjs -- node --test \"scripts/test/**/*.integration.test.mjs\"", "ci:local": "node scripts/ci-local.mjs" }, "engines": { diff --git a/packages/ai-features/package.json b/packages/ai-features/package.json index 50a7be80..764ed70f 100644 --- a/packages/ai-features/package.json +++ b/packages/ai-features/package.json @@ -19,7 +19,12 @@ ], "scripts": { "build": "tsc -p tsconfig.json", - "test": "tsc -p tsconfig.test.json && node --test \"dist-test/test/**/*.test.js\"", + "build:test": "tsc -p tsconfig.test.json", + "test:unit": "node ../../scripts/run-zero-skip.mjs -- node --test \"dist-test/test/**/!(*integration).test.js\"", + "test:live-provider": "npm run build:test && node ../../scripts/run-live-provider-tests.mjs all -- node --test \"dist-test/test/**/*integration.test.js\"", + "test:live-provider:openai": "npm run build:test && node ../../scripts/run-live-provider-tests.mjs openai -- node --test \"dist-test/test/**/*integration.test.js\"", + "test:live-provider:anthropic": "npm run build:test && node ../../scripts/run-live-provider-tests.mjs anthropic -- node --test \"dist-test/test/**/*integration.test.js\"", + "test": "npm run build:test && npm run test:unit", "lint": "tsc -p tsconfig.json --noEmit", "clean": "rm -rf dist dist-test" }, diff --git a/packages/ai-features/test/coach-integration.test.ts b/packages/ai-features/test/coach-integration.test.ts index 4a0e5da7..199886b6 100644 --- a/packages/ai-features/test/coach-integration.test.ts +++ b/packages/ai-features/test/coach-integration.test.ts @@ -1,7 +1,7 @@ /** * Env-gated integration test for `Coach`. * - * Skips without an API key, same pattern as the other five features. + * The live-provider runner registers this file only for the selected, provisioned provider. */ import { test, describe } from 'node:test'; @@ -26,8 +26,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('Coach integration (OpenAI)', () => { - test('real narrative with composed feature results', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('Coach integration (OpenAI)', () => { + test('real narrative with composed feature results', async () => { const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, defaultModel: 'gpt-4o-mini' }); const coach = new Coach({ engine: fakeEngine, ai }); @@ -41,9 +41,9 @@ describe('Coach integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('Coach integration (Anthropic)', () => { - test('real narrative with composed feature results', { skip: !anthropicKey }, async () => { - const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-3-5-sonnet-20241022' }); +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('Coach integration (Anthropic)', () => { + test('real narrative with composed feature results', async () => { + const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-sonnet-4-6' }); const coach = new Coach({ engine: fakeEngine, ai }); const result = await coach.coach({ fen: STARTPOS, move: 'a2a3' }); diff --git a/packages/ai-features/test/endgame-integration.test.ts b/packages/ai-features/test/endgame-integration.test.ts index 56ffc93e..59365d95 100644 --- a/packages/ai-features/test/endgame-integration.test.ts +++ b/packages/ai-features/test/endgame-integration.test.ts @@ -1,7 +1,7 @@ /** * Env-gated integration test for `EndgameTrainer`. * - * Skips without an API key, same pattern as the other four features. + * The live-provider runner registers this file only for the selected, provisioned provider. */ import { test, describe } from 'node:test'; @@ -30,8 +30,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('EndgameTrainer integration (OpenAI)', () => { - test('real narrative with engine-verified solution', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('EndgameTrainer integration (OpenAI)', () => { + test('real narrative with engine-verified solution', async () => { const db = new BundledEndgameDatabase(); const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, defaultModel: 'gpt-4o-mini' }); const trainer = new EndgameTrainer({ database: db, engine: fakeEngine, ai }); @@ -46,10 +46,10 @@ describe('EndgameTrainer integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('EndgameTrainer integration (Anthropic)', () => { - test('real coaching with engine-verified evaluation', { skip: !anthropicKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('EndgameTrainer integration (Anthropic)', () => { + test('real coaching with engine-verified evaluation', async () => { const db = new BundledEndgameDatabase(); - const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-3-5-sonnet-20241022' }); + const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-sonnet-4-6' }); const trainer = new EndgameTrainer({ database: db, engine: fakeEngine, ai }); const result = await trainer.evaluateAttempt({ diff --git a/packages/ai-features/test/integration.test.ts b/packages/ai-features/test/integration.test.ts index d3a8c72b..51e321d9 100644 --- a/packages/ai-features/test/integration.test.ts +++ b/packages/ai-features/test/integration.test.ts @@ -1,12 +1,10 @@ /** * Env-gated integration test for `MoveExplainer`. * - * Skips without an API key, exactly like M7's adapter tests. When the - * key is present, runs the real path against a real provider — proving - * the wiring works beyond fakes. + * The live-provider runner registers only the selected, provisioned provider and runs the + * real path against it, proving the wiring beyond fakes without creating skipped tests. * - * Run with: OPENAI_API_KEY=sk-... npm test - * ANTHROPIC_API_KEY=sk-ant-... npm test + * Run with the package's `test:live-provider:openai` or `test:live-provider:anthropic` script. */ import { test, describe } from 'node:test'; @@ -73,8 +71,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('MoveExplainer integration (OpenAI)', () => { - test('real completion with grounded citation', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('MoveExplainer integration (OpenAI)', () => { + test('real completion with grounded citation', async () => { const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, @@ -114,11 +112,11 @@ describe('MoveExplainer integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('MoveExplainer integration (Anthropic)', () => { - test('real completion with grounded citation', { skip: !anthropicKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('MoveExplainer integration (Anthropic)', () => { + test('real completion with grounded citation', async () => { const ai = new AnthropicAdapter({ apiKey: anthropicKey, - defaultModel: 'claude-3-5-sonnet-20241022', + defaultModel: 'claude-sonnet-4-6', }); const explainer = new MoveExplainer({ engine: fakeEngine, ai }); diff --git a/packages/ai-features/test/mistake-integration.test.ts b/packages/ai-features/test/mistake-integration.test.ts index ea48c14e..8e6cb8f7 100644 --- a/packages/ai-features/test/mistake-integration.test.ts +++ b/packages/ai-features/test/mistake-integration.test.ts @@ -1,13 +1,10 @@ /** * Env-gated integration test for `MistakePredictor`. * - * Skips without an API key, exactly like the MoveExplainer and - * PuzzleGenerator integration tests. When the key is present, runs - * the real path against a real provider — proving the wiring works - * beyond fakes. + * The live-provider runner registers only the selected, provisioned provider and runs the + * real path against it, proving the wiring beyond fakes without creating skipped tests. * - * Run with: OPENAI_API_KEY=sk-... npm test - * ANTHROPIC_API_KEY=sk-ant-... npm test + * Run with the package's `test:live-provider:openai` or `test:live-provider:anthropic` script. */ import { test, describe } from 'node:test'; @@ -86,8 +83,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('MistakePredictor integration (OpenAI)', () => { - test('real completion with engine-verified verdict', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('MistakePredictor integration (OpenAI)', () => { + test('real completion with engine-verified verdict', async () => { const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, @@ -126,11 +123,11 @@ describe('MistakePredictor integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('MistakePredictor integration (Anthropic)', () => { - test('real completion with engine-verified verdict', { skip: !anthropicKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('MistakePredictor integration (Anthropic)', () => { + test('real completion with engine-verified verdict', async () => { const ai = new AnthropicAdapter({ apiKey: anthropicKey, - defaultModel: 'claude-3-5-sonnet-20241022', + defaultModel: 'claude-sonnet-4-6', }); const predictor = new MistakePredictor({ engine: fakeEngine, ai }); diff --git a/packages/ai-features/test/opening-integration.test.ts b/packages/ai-features/test/opening-integration.test.ts index 569de1f9..e6948bcc 100644 --- a/packages/ai-features/test/opening-integration.test.ts +++ b/packages/ai-features/test/opening-integration.test.ts @@ -1,12 +1,10 @@ /** * Env-gated integration test for `OpeningExplorer`. * - * Skips without an API key, exactly like the other three features' - * integration tests. When the key is present, runs the real path - * against a real provider — proving the wiring works beyond fakes. + * The live-provider runner registers only the selected, provisioned provider and runs the + * real path against it, proving the wiring beyond fakes without creating skipped tests. * - * Run with: OPENAI_API_KEY=sk-... npm test - * ANTHROPIC_API_KEY=sk-ant-... npm test + * Run with the package's `test:live-provider:openai` or `test:live-provider:anthropic` script. */ import { test, describe } from 'node:test'; @@ -23,8 +21,8 @@ const TEST_MOVES = ['e2e4', 'c7c5', 'g1f3', 'd7d6', 'd2d4', 'c5d4', 'f3d4', 'g8f const openaiKey = process.env['OPENAI_API_KEY']; -describe('OpeningExplorer integration (OpenAI)', () => { - test('real narrative with opening identification', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('OpeningExplorer integration (OpenAI)', () => { + test('real narrative with opening identification', async () => { const db = new BundledOpeningDatabase(); const ai = new OpenAiCompatibleAdapter({ id: 'openai', @@ -49,12 +47,12 @@ describe('OpeningExplorer integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('OpeningExplorer integration (Anthropic)', () => { - test('real narrative with opening identification', { skip: !anthropicKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('OpeningExplorer integration (Anthropic)', () => { + test('real narrative with opening identification', async () => { const db = new BundledOpeningDatabase(); const ai = new AnthropicAdapter({ apiKey: anthropicKey, - defaultModel: 'claude-3-5-sonnet-20241022', + defaultModel: 'claude-sonnet-4-6', }); const explorer = new OpeningExplorer({ database: db, ai }); diff --git a/packages/ai-features/test/puzzle-integration.test.ts b/packages/ai-features/test/puzzle-integration.test.ts index f170f306..998d275d 100644 --- a/packages/ai-features/test/puzzle-integration.test.ts +++ b/packages/ai-features/test/puzzle-integration.test.ts @@ -1,13 +1,10 @@ /** * Env-gated integration test for `PuzzleGenerator`. * - * Skips without an API key, exactly like M7's adapter tests and the - * MoveExplainer integration test. When the key is present, runs the - * real path against a real provider — proving the wiring works beyond - * fakes. + * The live-provider runner registers only the selected, provisioned provider and runs the + * real path against it, proving the wiring beyond fakes without creating skipped tests. * - * Run with: OPENAI_API_KEY=sk-... npm test - * ANTHROPIC_API_KEY=sk-ant-... npm test + * Run with the package's `test:live-provider:openai` or `test:live-provider:anthropic` script. */ import { test, describe } from 'node:test'; @@ -89,8 +86,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('PuzzleGenerator integration (OpenAI)', () => { - test('real completion with engine-verified puzzle', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('PuzzleGenerator integration (OpenAI)', () => { + test('real completion with engine-verified puzzle', async () => { const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, @@ -126,11 +123,11 @@ describe('PuzzleGenerator integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('PuzzleGenerator integration (Anthropic)', () => { - test('real completion with engine-verified puzzle', { skip: !anthropicKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('PuzzleGenerator integration (Anthropic)', () => { + test('real completion with engine-verified puzzle', async () => { const ai = new AnthropicAdapter({ apiKey: anthropicKey, - defaultModel: 'claude-3-5-sonnet-20241022', + defaultModel: 'claude-sonnet-4-6', }); const generator = new PuzzleGenerator({ engine: fakeEngine, ai }); diff --git a/packages/ai-features/test/study-integration.test.ts b/packages/ai-features/test/study-integration.test.ts index abb214fc..a34cfb6b 100644 --- a/packages/ai-features/test/study-integration.test.ts +++ b/packages/ai-features/test/study-integration.test.ts @@ -1,7 +1,7 @@ /** * Env-gated integration test for `StudyPartner`. * - * Skips without an API key, same pattern as the other six features. + * The live-provider runner registers this file only for the selected, provisioned provider. */ import { test, describe } from 'node:test'; @@ -25,8 +25,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('StudyPartner integration (OpenAI)', () => { - test('real session with narrative', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('StudyPartner integration (OpenAI)', () => { + test('real session with narrative', async () => { const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, defaultModel: 'gpt-4o-mini' }); const coach = new Coach({ engine: fakeEngine, ai }); const store = new InMemoryStudySessionStore(); @@ -46,9 +46,9 @@ describe('StudyPartner integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('StudyPartner integration (Anthropic)', () => { - test('real session with narrative', { skip: !anthropicKey }, async () => { - const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-3-5-sonnet-20241022' }); +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('StudyPartner integration (Anthropic)', () => { + test('real session with narrative', async () => { + const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-sonnet-4-6' }); const coach = new Coach({ engine: fakeEngine, ai }); const store = new InMemoryStudySessionStore(); const partner = new StudyPartner({ store, coach, ai, idGenerator: () => 'int-anthropic' }); diff --git a/packages/ai-features/test/tournament-commentator-integration.test.ts b/packages/ai-features/test/tournament-commentator-integration.test.ts index f35038af..7435d395 100644 --- a/packages/ai-features/test/tournament-commentator-integration.test.ts +++ b/packages/ai-features/test/tournament-commentator-integration.test.ts @@ -1,7 +1,7 @@ /** * Live integration test for `TournamentCommentator`. * - * Gated by `GAMBIT_TEST_INTEGRATION=1`. + * Selected by `GAMBIT_LIVE_PROVIDER=openai` through the live-provider runner. * Uses a real `OpenAiCompatibleAdapter` (requires `OPENAI_API_KEY`) and a * mock `AnalysisProvider` to verify wiring to a real LLM. */ @@ -50,7 +50,7 @@ const fakeEngine: AnalysisProvider = { }, }; -describe('TournamentCommentator (integration)', { skip: !process.env.GAMBIT_TEST_INTEGRATION }, () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('TournamentCommentator (integration)', () => { let commentator: TournamentCommentator; before(async () => { diff --git a/packages/ai-features/test/voice-coach-integration.test.ts b/packages/ai-features/test/voice-coach-integration.test.ts index 83cc4058..92a89498 100644 --- a/packages/ai-features/test/voice-coach-integration.test.ts +++ b/packages/ai-features/test/voice-coach-integration.test.ts @@ -1,7 +1,7 @@ /** * Env-gated integration test for `VoiceCoach`. * - * Skips without an API key, same pattern as the other seven features. + * The live-provider runner registers this file only for the selected, provisioned provider. */ import { test, describe } from 'node:test'; @@ -25,8 +25,8 @@ const fakeEngine: AnalysisProvider = { const openaiKey = process.env['OPENAI_API_KEY']; -describe('VoiceCoach integration (OpenAI)', () => { - test('real narrative smoothing with spoken segments', { skip: !openaiKey }, async () => { +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') describe('VoiceCoach integration (OpenAI)', () => { + test('real narrative smoothing with spoken segments', async () => { const ai = new OpenAiCompatibleAdapter({ id: 'openai', apiKey: openaiKey, defaultModel: 'gpt-4o-mini' }); const coach = new Coach({ engine: fakeEngine }); const voiceCoach = new VoiceCoach({ coach, ai }); @@ -43,9 +43,9 @@ describe('VoiceCoach integration (OpenAI)', () => { const anthropicKey = process.env['ANTHROPIC_API_KEY']; -describe('VoiceCoach integration (Anthropic)', () => { - test('real narrative smoothing with spoken segments', { skip: !anthropicKey }, async () => { - const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-3-5-sonnet-20241022' }); +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') describe('VoiceCoach integration (Anthropic)', () => { + test('real narrative smoothing with spoken segments', async () => { + const ai = new AnthropicAdapter({ apiKey: anthropicKey, defaultModel: 'claude-sonnet-4-6' }); const coach = new Coach({ engine: fakeEngine }); const voiceCoach = new VoiceCoach({ coach, ai }); diff --git a/packages/ai-orchestrator/package.json b/packages/ai-orchestrator/package.json index 89c2f3e0..39798b86 100644 --- a/packages/ai-orchestrator/package.json +++ b/packages/ai-orchestrator/package.json @@ -19,7 +19,12 @@ ], "scripts": { "build": "tsc -p tsconfig.json", - "test": "tsc -p tsconfig.test.json && node --test \"dist-test/test/**/*.test.js\"", + "build:test": "tsc -p tsconfig.test.json", + "test:unit": "node ../../scripts/run-zero-skip.mjs -- node --test \"dist-test/test/**/!(*integration).test.js\"", + "test:live-provider": "npm run build:test && node ../../scripts/run-live-provider-tests.mjs all -- node --test \"dist-test/test/**/*integration.test.js\"", + "test:live-provider:openai": "npm run build:test && node ../../scripts/run-live-provider-tests.mjs openai -- node --test \"dist-test/test/**/*integration.test.js\"", + "test:live-provider:anthropic": "npm run build:test && node ../../scripts/run-live-provider-tests.mjs anthropic -- node --test \"dist-test/test/**/*integration.test.js\"", + "test": "npm run build:test && npm run test:unit", "lint": "tsc -p tsconfig.json --noEmit", "clean": "rm -rf dist dist-test" }, diff --git a/packages/ai-orchestrator/test/adapters-live.integration.test.ts b/packages/ai-orchestrator/test/adapters-live.integration.test.ts new file mode 100644 index 00000000..bf758959 --- /dev/null +++ b/packages/ai-orchestrator/test/adapters-live.integration.test.ts @@ -0,0 +1,34 @@ +import { test } from 'node:test'; +import * as assert from 'node:assert/strict'; +import { OpenAiCompatibleAdapter, AnthropicAdapter } from '../src/index.js'; + +const openaiKey = process.env['OPENAI_API_KEY']; +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'openai') test('OpenAI adapter: real completion', async () => { + const adapter = new OpenAiCompatibleAdapter({ + id: 'openai', + apiKey: openaiKey, + defaultModel: 'gpt-4o-mini', + }); + const response = await adapter.complete({ + task: 'general', + messages: [{ role: 'user', content: 'What is 2+2? Reply with just the number.' }], + maxTokens: 10, + }); + assert.ok(response.content); + assert.equal(response.providerId, 'openai'); +}); + +const anthropicKey = process.env['ANTHROPIC_API_KEY']; +if (process.env['GAMBIT_LIVE_PROVIDER'] === 'anthropic') test('Anthropic adapter: real completion', async () => { + const adapter = new AnthropicAdapter({ + apiKey: anthropicKey, + defaultModel: 'claude-sonnet-4-6', + }); + const response = await adapter.complete({ + task: 'general', + messages: [{ role: 'user', content: 'What is 2+2? Reply with just the number.' }], + maxTokens: 10, + }); + assert.ok(response.content); + assert.equal(response.providerId, 'anthropic'); +}); diff --git a/packages/ai-orchestrator/test/adapters.test.ts b/packages/ai-orchestrator/test/adapters.test.ts index 3f9cbfc4..15ce9478 100644 --- a/packages/ai-orchestrator/test/adapters.test.ts +++ b/packages/ai-orchestrator/test/adapters.test.ts @@ -1,12 +1,8 @@ /** - * Integration tests for the OpenAI-compatible and Anthropic adapters. + * Hermetic tests for the OpenAI-compatible and Anthropic adapters. * - * These tests are env-gated: they skip unless the relevant API key - * is present in the environment, exactly like the Postgres-gated - * persistence tests (gated on `DATABASE_URL`). - * - * Run with: OPENAI_API_KEY=sk-... npm test - * ANTHROPIC_API_KEY=sk-ant-... npm test + * Live provider contracts are isolated in `adapters-live.integration.test.ts` and run through the + * provider-specific zero-skip package commands. */ import { test, describe } from 'node:test'; @@ -173,25 +169,8 @@ describe('OpenAiCompatibleAdapter', () => { }); }); -// Env-gated integration test (skips without OPENAI_API_KEY) -const openaiKey = process.env['OPENAI_API_KEY']; -test('OpenAI adapter: real completion', { skip: !openaiKey }, async () => { - const adapter = new OpenAiCompatibleAdapter({ - id: 'openai', - apiKey: openaiKey, - defaultModel: 'gpt-4o-mini', - }); - const response = await adapter.complete({ - task: 'general', - messages: [{ role: 'user', content: 'What is 2+2? Reply with just the number.' }], - maxTokens: 10, - }); - assert.ok(response.content); - assert.equal(response.providerId, 'openai'); -}); - // --------------------------------------------------------------------------- -// Anthropic adapter (env-gated) +// Anthropic adapter // --------------------------------------------------------------------------- describe('AnthropicAdapter', () => { @@ -208,19 +187,3 @@ describe('AnthropicAdapter', () => { await assert.rejects(adapter.embed({ input: 'test' }), /does not support embeddings/); }); }); - -// Env-gated integration test (skips without ANTHROPIC_API_KEY) -const anthropicKey = process.env['ANTHROPIC_API_KEY']; -test('Anthropic adapter: real completion', { skip: !anthropicKey }, async () => { - const adapter = new AnthropicAdapter({ - apiKey: anthropicKey, - defaultModel: 'claude-3-5-sonnet-20241022', - }); - const response = await adapter.complete({ - task: 'general', - messages: [{ role: 'user', content: 'What is 2+2? Reply with just the number.' }], - maxTokens: 10, - }); - assert.ok(response.content); - assert.equal(response.providerId, 'anthropic'); -}); diff --git a/packages/api/package.json b/packages/api/package.json index 37bf3112..675fef3d 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -22,9 +22,13 @@ ], "scripts": { "build": "tsc -p tsconfig.json", - "test": "tsc -p tsconfig.test.json && node --test --test-concurrency=1 \"dist-test/test/**/*.test.js\"", - "test:analysis-smoke": "tsc -p tsconfig.test.json && node --test dist-test/test/analysis-stockfish-smoke.test.js dist-test/test/analysis-fairy-threecheck-smoke.test.js dist-test/test/analysis-real-stack.test.js", - "test:diagnostics:abort": "tsc -p tsconfig.test.json && node --test dist-test/test/diagnostics/signature-b-preload-abort.diag.js", + "build:test": "tsc -p tsconfig.test.json", + "test:unit": "node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 \"dist-test/test/**/!(analysis-*smoke|analysis-real-stack|*.integration|*.posix).test.js\"", + "test:posix": "npm run build:test && node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 \"dist-test/test/**/*.posix.test.js\"", + "test:integration:postgres": "npm run build:test && node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 \"dist-test/test/**/*.integration.test.js\"", + "test:analysis-smoke": "tsc -p tsconfig.test.json && node ../../scripts/run-zero-skip.mjs -- node --test dist-test/test/analysis-stockfish-smoke.test.js dist-test/test/analysis-fairy-threecheck-smoke.test.js dist-test/test/analysis-real-stack.test.js", + "test": "npm run build:test && npm run test:unit", + "test:diagnostics:abort": "tsc -p tsconfig.test.json && node ../../scripts/run-zero-skip.mjs -- node --test dist-test/test/diagnostics/signature-b-preload-abort.diag.js", "test:diagnostics:signature-b": "tsc -p tsconfig.test.json && node ./test/diagnostics/run-signature-b-pass.mjs", "lint": "tsc -p tsconfig.json --noEmit", "openapi": "node dist/scripts/generate-openapi.js", diff --git a/packages/api/test/diagnostics/signature-b-correlate.posix.test.ts b/packages/api/test/diagnostics/signature-b-correlate.posix.test.ts new file mode 100644 index 00000000..dee80f9a --- /dev/null +++ b/packages/api/test/diagnostics/signature-b-correlate.posix.test.ts @@ -0,0 +1,141 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +const DIAGNOSTICS_DIR = path.resolve(__dirname, '../../../test/diagnostics'); + +/** Runs the pass script and returns its result, isolated from this suite's own test context. */ +function runPass(args: string[], cwd?: string): ReturnType { + return spawnSync(process.execPath, [path.join(DIAGNOSTICS_DIR, 'run-signature-b-pass.mjs'), ...args], { + cwd: cwd ?? path.resolve(DIAGNOSTICS_DIR, '../..'), + encoding: 'utf8', + env: { ...process.env, NODE_TEST_CONTEXT: undefined }, + }); +} + +/** + * Read a worker pid once the file actually holds one. + */ +async function readWorkerPid(pidFile: string, timeoutMs = 20_000): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + const pid = Number(fs.readFileSync(pidFile, 'utf8').trim()); + if (Number.isInteger(pid) && pid > 0) return pid; + } catch { + /* not written yet */ + } + if (Date.now() >= deadline) return null; + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +/** + * Kill a pid outright, tolerating one that is already gone. + */ +function reap(pid: number | null): void { + if (pid === null) return; + try { + process.kill(pid, 'SIGKILL'); + } catch { + /* already gone */ + } +} + +/** + * Wait for a pid to disappear, or give up. + */ +async function waitForExit(pid: number, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + for (;;) { + try { + process.kill(pid, 0); + } catch { + return true; + } + if (Date.now() >= deadline) return false; + await new Promise((resolve) => setTimeout(resolve, 50)); + } +} + +/** A test file that finishes immediately. */ +function trivialTarget(dir: string): string { + const file = path.join(dir, 'noop.test.cjs'); + fs.writeFileSync(file, "require('node:test').test('noop', () => {});\n"); + return file; +} + +test('pass runner: interrupting the pass takes the detached test tree with it', async () => { + // Detaching the run is what makes its group killable, and it is also what stops a terminal Ctrl-C + // reaching it: the runner is no longer in the foreground process group. Without the handlers this + // asserts, interrupting a pass would leave node --test and every per-file worker running against + // the suite database. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sigb-signal-')); + try { + const pidFile = path.join(dir, 'child.pid'); + const target = path.join(dir, 'blocks.test.cjs'); + fs.writeFileSync( + target, + `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));\n` + + "require('node:test').test('blocks', async () => {\n" + + ' await new Promise((r) => setTimeout(r, 30000));\n' + + '});\n', + ); + + let workerPid: number | null = null; + const pass = spawn( + process.execPath, + [path.join(DIAGNOSTICS_DIR, 'run-signature-b-pass.mjs'), '--runs', '1', '--target', target, '--out', path.join(dir, 'out')], + { cwd: path.resolve(DIAGNOSTICS_DIR, '../..'), env: { ...process.env, NODE_TEST_CONTEXT: undefined }, stdio: 'ignore' }, + ); + + try { + // Wait for the worker to have written a usable pid, so the interrupt has something to clean + // up and the liveness probe addresses the worker rather than this process group. + workerPid = await readWorkerPid(pidFile); + assert.ok(workerPid !== null, 'the per-file worker started and recorded its pid'); + + pass.kill('SIGTERM'); + await new Promise((resolve) => pass.on('close', resolve)); + + assert.equal( + await waitForExit(workerPid), + true, + 'the detached per-file worker must not survive the interrupted pass', + ); + // Same reason as the ceiling test: once the pid is proven gone it belongs to nobody, and + // reaping it later could kill an unrelated process the OS gave that number to. + workerPid = null; + } finally { + // Two ways this test could leak the tree it started: the worker never appears, so the + // assertion throws before the interrupt is sent; or the cleanup being asserted did not happen, + // so the worker is still sleeping. Both are covered here rather than in the happy path. + pass.kill('SIGKILL'); + reap(workerPid); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('pass runner: an existing group-readable --out is secured or refused', () => { + // `mkdirSync`'s mode applies only when it creates the directory, so an existing `--out` keeps + // whatever permissions it had — and a captured run's artifacts would land in a shared directory. + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sigb-perm-')); + try { + const out = path.join(dir, 'shared'); + fs.mkdirSync(out, { recursive: true }); + fs.chmodSync(out, 0o777); + + const result = runPass(['--runs', '1', '--target', trivialTarget(dir), '--out', out]); + + assert.equal(result.status, 0, 'a securable directory is narrowed rather than refused'); + assert.equal(fs.statSync(out).mode & 0o077, 0, 'and it is owner-only before anything is written into it'); + assert.match(`${result.stdout}`, /narrowed/, 'and the narrowing is stated rather than done silently'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); diff --git a/packages/api/test/diagnostics/signature-b-correlate.test.ts b/packages/api/test/diagnostics/signature-b-correlate.test.ts index 0e3568a6..7664de62 100644 --- a/packages/api/test/diagnostics/signature-b-correlate.test.ts +++ b/packages/api/test/diagnostics/signature-b-correlate.test.ts @@ -525,63 +525,6 @@ test('pass runner: a completed run is not reported as timed out', () => { } }); -test('pass runner: interrupting the pass takes the detached test tree with it', { - skip: process.platform === 'win32' - ? 'SIGTERM on Windows terminates without running handlers, and libuv already reaps the tree there' - : false, -}, async () => { - // Detaching the run is what makes its group killable, and it is also what stops a terminal Ctrl-C - // reaching it: the runner is no longer in the foreground process group. Without the handlers this - // asserts, interrupting a pass would leave node --test and every per-file worker running against - // the suite database. - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sigb-signal-')); - try { - const pidFile = path.join(dir, 'child.pid'); - const target = path.join(dir, 'blocks.test.cjs'); - fs.writeFileSync( - target, - `require('node:fs').writeFileSync(${JSON.stringify(pidFile)}, String(process.pid));\n` + - "require('node:test').test('blocks', async () => {\n" + - ' await new Promise((r) => setTimeout(r, 30000));\n' + - '});\n', - ); - - let workerPid: number | null = null; - const pass = spawn( - process.execPath, - [path.join(DIAGNOSTICS_DIR, 'run-signature-b-pass.mjs'), '--runs', '1', '--target', target, '--out', path.join(dir, 'out')], - { cwd: path.resolve(DIAGNOSTICS_DIR, '../..'), env: { ...process.env, NODE_TEST_CONTEXT: undefined }, stdio: 'ignore' }, - ); - - try { - // Wait for the worker to have written a usable pid, so the interrupt has something to clean - // up and the liveness probe addresses the worker rather than this process group. - workerPid = await readWorkerPid(pidFile); - assert.ok(workerPid !== null, 'the per-file worker started and recorded its pid'); - - pass.kill('SIGTERM'); - await new Promise((resolve) => pass.on('close', resolve)); - - assert.equal( - await waitForExit(workerPid), - true, - 'the detached per-file worker must not survive the interrupted pass', - ); - // Same reason as the ceiling test: once the pid is proven gone it belongs to nobody, and - // reaping it later could kill an unrelated process the OS gave that number to. - workerPid = null; - } finally { - // Two ways this test could leak the tree it started: the worker never appears, so the - // assertion throws before the interrupt is sent; or the cleanup being asserted did not happen, - // so the worker is still sleeping. Both are covered here rather than in the happy path. - pass.kill('SIGKILL'); - reap(workerPid); - } - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - test('pass runner: a capture keeps the normalised record and discards the raw TAP', () => { // The raw TAP is a full transcript of whatever the suite printed. Once `capture.json` holds the // enumerated fields, keeping the transcript retains arbitrary test output for no diagnostic gain. @@ -608,25 +551,6 @@ test('pass runner: a capture keeps the normalised record and discards the raw TA } }); -test('pass runner: an existing group-readable --out is secured or refused', { skip: process.platform === 'win32' ? 'POSIX permission bits are not meaningful on Windows; this guarantee is offered on POSIX only' : false }, () => { - // `mkdirSync`'s mode applies only when it creates the directory, so an existing `--out` keeps - // whatever permissions it had — and a captured run's artifacts would land in a shared directory. - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'sigb-perm-')); - try { - const out = path.join(dir, 'shared'); - fs.mkdirSync(out, { recursive: true }); - fs.chmodSync(out, 0o777); - - const result = runPass(['--runs', '1', '--target', trivialTarget(dir), '--out', out]); - - assert.equal(result.status, 0, 'a securable directory is narrowed rather than refused'); - assert.equal(fs.statSync(out).mode & 0o077, 0, 'and it is owner-only before anything is written into it'); - assert.match(`${result.stdout}`, /narrowed/, 'and the narrowing is stated rather than done silently'); - } finally { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - test('pass runner: an experiment that runs nothing is refused, not reported clean', () => { const runner = path.join(DIAGNOSTICS_DIR, 'run-signature-b-pass.mjs'); const attempt = (runs: string): ReturnType => diff --git a/packages/persistence/package.json b/packages/persistence/package.json index b2593c68..79ce8880 100644 --- a/packages/persistence/package.json +++ b/packages/persistence/package.json @@ -30,7 +30,10 @@ ], "scripts": { "build": "tsc -p tsconfig.json", - "test": "tsc -p tsconfig.test.json && node --test --test-concurrency=1 \"dist-test/test/**/*.test.js\"", + "build:test": "tsc -p tsconfig.test.json", + "test:unit": "node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 \"dist-test/test/**/!(*.integration).test.js\"", + "test:integration:postgres": "npm run build:test && node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 \"dist-test/test/**/*.integration.test.js\"", + "test": "npm run build:test && npm run test:unit", "lint": "tsc -p tsconfig.json --noEmit", "migrate": "node dist/pg/migrate-cli.js", "clean": "rm -rf dist dist-test" diff --git a/packages/persistence/test/pg/identity-tokens.test.ts b/packages/persistence/test/pg/identity-tokens.integration.test.ts similarity index 100% rename from packages/persistence/test/pg/identity-tokens.test.ts rename to packages/persistence/test/pg/identity-tokens.integration.test.ts diff --git a/packages/web/playwright.config.ts b/packages/web/playwright.config.ts index 2763e873..d1e2a10f 100644 --- a/packages/web/playwright.config.ts +++ b/packages/web/playwright.config.ts @@ -5,9 +5,8 @@ * npm run e2e # static/offline specs (vite preview only) * GAMBIT_E2E_BACKEND=1 npm run e2e # all specs (starts e2e harness + vite preview) * - * Backend-dependent specs (game-vs-bot, game-vs-human) are gated with - * `test.skip(!process.env.GAMBIT_E2E_BACKEND, ...)` so `npm run e2e` - * without backends only runs the static/offline specs. + * Backend-dependent specs are excluded during offline discovery. Their own + * `test.skip` guards remain a safety net for direct file invocations. * * When GAMBIT_E2E_BACKEND=1, Playwright starts and health-checks both the * e2e harness and Vite preview. Keeping them as separate managed processes is @@ -19,13 +18,40 @@ */ import type { PlaywrightTestConfig } from '@playwright/test'; import { cpus } from 'node:os'; +import { fileURLToPath } from 'node:url'; const isBackend = !!process.env['GAMBIT_E2E_BACKEND']; +const zeroSkipReporter = fileURLToPath( + new URL('../../scripts/playwright-zero-skip-reporter.mjs', import.meta.url) +); +const backendSpecs = [ + 'account-security-sessions.spec.ts', + 'achievements.spec.ts', + 'analysis.spec.ts', + 'forum.spec.ts', + 'game-actions.spec.ts', + 'game-keyboard.spec.ts', + 'game-lifecycle.spec.ts', + 'game-presence.spec.ts', + 'game-responsive.spec.ts', + 'game-vs-bot.spec.ts', + 'game-vs-human.spec.ts', + 'learning.spec.ts', + 'messages.spec.ts', + 'play-vs-computer.spec.ts', + 'search.spec.ts', + 'seek-acceptance.spec.ts', + 'studies.spec.ts', + 'teams.spec.ts', + 'tournaments.spec.ts', +]; const config: PlaywrightTestConfig = { testDir: './e2e', + testIgnore: isBackend ? [] : backendSpecs.map((name) => `**/${name}`), timeout: 300_000, retries: 1, + reporter: [['list'], [zeroSkipReporter]], // Ceiling, not a fixed count: pinning `workers: 4` would RAISE parallelism on a 2-core CI // runner, which is the opposite of the fix. The backend-gated suite drives one shared single-process // `e2e-harness` and one vite preview server, and unbounded local parallelism starves them. diff --git a/scripts/check-test-topology.mjs b/scripts/check-test-topology.mjs new file mode 100644 index 00000000..3bf23d23 --- /dev/null +++ b/scripts/check-test-topology.mjs @@ -0,0 +1,1030 @@ +import { existsSync, readdirSync, readFileSync, unlinkSync, writeFileSync } from 'node:fs'; +import { dirname, join, posix, relative, resolve, sep } from 'node:path'; +import { spawnSync } from 'node:child_process'; +import { createRequire } from 'node:module'; +import { randomBytes } from 'node:crypto'; + +const REPO_ROOT = resolve(process.cwd()); + +/** + * Strips single-line and multi-line comments from JS/TS code while preserving string literals. + * + * @param {string} code + * @returns {string} + */ +export function stripComments(code) { + return code.replace( + /("(?:\\[\s\S]|[^"\\])*"|'(?:\\[\s\S]|[^'\\])*'|`(?:\\[\s\S]|[^`\\])*`)|\/\*[\s\S]*?\*\/|\/\/[^\r\n]*/g, + (match, str) => (str ? str : '') + ); +} + +/** + * Extracts a static string literal starting at index 0 of str. + * Supports single quotes, double quotes, and template literals without interpolation (${...). + * + * @param {string} str + * @returns {{ value: string, endIndex: number } | null} + */ +export function extractStringLiteralAtStart(str) { + const quote = str[0]; + if (quote !== "'" && quote !== '"' && quote !== '`') return null; + let i = 1; + let escaped = false; + while (i < str.length) { + const ch = str[i]; + if (escaped) { + escaped = false; + } else if (ch === '\\') { + escaped = true; + } else if (ch === quote) { + const content = str.slice(1, i); + if (quote === '`' && content.includes('${')) { + return null; + } + return { + value: content, + endIndex: i + 1, + }; + } + i++; + } + return null; +} + +/** + * Parses an array literal of static string literals (e.g. ['**\/*.spec.ts', '**\/*.test.ts']). + * Fails closed (returns null) if any element is non-literal or unparseable. + * + * @param {string} str + * @returns {string[] | null} + */ +export function parseStringLiteralArray(str) { + if (!str.startsWith('[')) return null; + let i = 1; + const elements = []; + while (i < str.length) { + while (i < str.length && /\s/.test(str[i])) i++; + if (i >= str.length) return null; + if (str[i] === ']') { + return elements; + } + const literal = extractStringLiteralAtStart(str.slice(i)); + if (!literal) { + return null; + } + elements.push(literal.value); + i += literal.endIndex; + while (i < str.length && /\s/.test(str[i])) i++; + if (i < str.length && str[i] === ',') { + i++; + } else if (i < str.length && str[i] === ']') { + return elements; + } else { + return null; + } + } + return null; +} + +export function parseStaticStringLiteral(raw) { + const trimmed = raw.trim(); + const match = trimmed.match(/^(['"`])([\s\S]*)\1$/); + if (!match) return null; + const quote = match[1]; + const content = match[2]; + if (quote === '`' && content.includes('${')) { + return null; + } + return content; +} + +/** + * Locates the opening and closing curly braces of an object literal in str starting from fromIndex. + * + * @param {string} str - Source string. + * @param {number} fromIndex - Index to search from. + * @returns {string | null} The object literal body (between { and }), or null if not found. + */ +export function extractObjectBody(str, fromIndex = 0) { + let openBrace = -1; + let i = fromIndex; + while (i < str.length) { + const lit = extractStringLiteralAtStart(str.slice(i)); + if (lit) { + i += lit.endIndex; + continue; + } + if (str[i] === '{') { + openBrace = i; + break; + } + i++; + } + if (openBrace === -1) return null; + + let braceDepth = 0; + i = openBrace; + while (i < str.length) { + const lit = extractStringLiteralAtStart(str.slice(i)); + if (lit) { + i += lit.endIndex; + continue; + } + if (str[i] === '{') { + braceDepth++; + } else if (str[i] === '}') { + braceDepth--; + if (braceDepth === 0) { + return str.slice(openBrace + 1, i); + } + } + i++; + } + return null; +} + +/** + * Finds the body of the exported Playwright configuration object. + * + * @param {string} stripped - Source code with comments removed. + * @param {string} configRel - Relative path for diagnostics. + * @returns {string} The inner body of the exported configuration object. + */ +export function findExportedPlaywrightConfigBody(stripped, configRel) { + const exportMatch = stripped.match(/(?:export\s+default|module\.exports\s*=)\s*([\s\S]*)/); + if (exportMatch) { + const afterExport = exportMatch[1].trimStart(); + const idMatch = afterExport.match(/^([A-Za-z0-9_$]+)\s*;?/); + if (idMatch && idMatch[1] !== 'defineConfig') { + const varName = idMatch[1]; + const declRegex = new RegExp(`(?:const|let|var)\\s+${varName}\\s*(?::\\s*[^=]+)?=\\s*([\\s\\S]*)`); + const declMatch = stripped.match(declRegex); + if (!declMatch) { + throw new Error( + `Cannot mechanically resolve Playwright configuration object in ${configRel}: exported identifier '${varName}' declaration cannot be statically resolved. Topology validation must fail closed.` + ); + } + const body = extractObjectBody(declMatch[1], 0); + if (body === null) { + throw new Error( + `Cannot mechanically resolve Playwright configuration object in ${configRel}: exported object structure cannot be statically resolved. Topology validation must fail closed.` + ); + } + return body; + } + + const body = extractObjectBody(afterExport, 0); + if (body === null) { + throw new Error( + `Cannot mechanically resolve Playwright configuration object in ${configRel}: exported object structure cannot be statically resolved. Topology validation must fail closed.` + ); + } + return body; + } + + const body = extractObjectBody(stripped, 0); + if (body !== null) { + return body; + } + return stripped; +} + +/** + * Scans an object literal body for direct properties at depth 0. + * Ignores properties in nested objects (like projects: [{ ... }]) or string literals. + * + * @param {string} objectBody + * @returns {Map} Map of direct property keys to their raw value expressions. + */ +export function extractDirectProperties(objectBody) { + const properties = new Map(); + let depth = { brace: 0, bracket: 0, paren: 0 }; + let idx = 0; + + while (idx < objectBody.length) { + const literal = extractStringLiteralAtStart(objectBody.slice(idx)); + if (literal) { + idx += literal.endIndex; + continue; + } + + const ch = objectBody[idx]; + if (ch === '{') { depth.brace++; idx++; continue; } + if (ch === '}') { depth.brace--; idx++; continue; } + if (ch === '[') { depth.bracket++; idx++; continue; } + if (ch === ']') { depth.bracket--; idx++; continue; } + if (ch === '(') { depth.paren++; idx++; continue; } + if (ch === ')') { depth.paren--; idx++; continue; } + + if (depth.brace === 0 && depth.bracket === 0 && depth.paren === 0) { + const propMatch = objectBody.slice(idx).match(/^(?:([A-Za-z0-9_$]+)|['"]([A-Za-z0-9_$]+)['"])\s*:\s*/); + if (propMatch) { + const key = propMatch[1] || propMatch[2]; + idx += propMatch[0].length; + const valStart = idx; + let valDepth = { brace: 0, bracket: 0, paren: 0 }; + while (idx < objectBody.length) { + const valLit = extractStringLiteralAtStart(objectBody.slice(idx)); + if (valLit) { + idx += valLit.endIndex; + continue; + } + const vch = objectBody[idx]; + if (vch === '{') { valDepth.brace++; idx++; continue; } + if (vch === '}') { valDepth.brace--; idx++; continue; } + if (vch === '[') { valDepth.bracket++; idx++; continue; } + if (vch === ']') { valDepth.bracket--; idx++; continue; } + if (vch === '(') { valDepth.paren++; idx++; continue; } + if (vch === ')') { valDepth.paren--; idx++; continue; } + + if (vch === ',' && valDepth.brace === 0 && valDepth.bracket === 0 && valDepth.paren === 0) { + break; + } + idx++; + } + const rawVal = objectBody.slice(valStart, idx).trim(); + properties.set(key, rawVal); + if (idx < objectBody.length && objectBody[idx] === ',') { + idx++; + } + continue; + } + } + + idx++; + } + + return properties; +} + +/** + * Recursively collects discovered test files from Playwright JSON report suites. + * + * @param {object} suite - Suite or project object from Playwright JSON report. + * @param {string} rootDir - Root directory reported by Playwright. + * @param {Set} discovered - Set of repository-relative POSIX paths. + * @param {string} repoRoot - Repository root directory. + */ +function collectPlaywrightFiles(suite, rootDir, discovered, repoRoot) { + if (!suite) return; + if (typeof suite.file === 'string' && suite.file.length > 0) { + const absPath = resolve(rootDir, suite.file); + const relPath = relative(repoRoot, absPath).split(sep).join('/'); + discovered.add(relPath); + } + if (Array.isArray(suite.suites)) { + for (const child of suite.suites) { + collectPlaywrightFiles(child, rootDir, discovered, repoRoot); + } + } +} + +/** + * Derives the exact set of reachable test files directly from Playwright's discovery engine + * (`playwright test --list --reporter=json`). + * + * Evaluates real Playwright configuration including: + * - Top-level and project-specific `testDir` + * - Top-level and project-specific `testMatch` + * - Top-level and project-specific `testIgnore` + * - Object spreads, variables, and dynamic configuration + * + * Fails closed: if Playwright configuration cannot be resolved, compiled, or executed, + * throws an explicit Error instead of substituting false-green assumptions. + * + * @param {string} [manifestDir='packages/web'] - Directory containing Playwright config and package.json. + * @param {object} [options={}] - Options (root, backend, playwrightConfigOverrides, playwrightDiscoveredCache, scriptCmd). + * @returns {Set} Set of repository-relative POSIX file paths discovered by Playwright. + */ +export function getPlaywrightDiscoveredFiles(manifestDir = 'packages/web', options = {}) { + const root = options.root || REPO_ROOT; + const manifestDirFull = resolve(root, manifestDir); + + const cache = options.playwrightDiscoveredCache; + const cacheKey = `${manifestDirFull}::${options.backend === false ? 'offline' : 'backend'}::${options.playwrightConfigOverrides ? JSON.stringify(options.playwrightConfigOverrides) : ''}::${options.scriptCmd || ''}`; + if (cache && cache.has(cacheKey)) { + return cache.get(cacheKey); + } + + const req = createRequire(import.meta.url); + let cliPath; + try { + cliPath = req.resolve('@playwright/test/cli', { + paths: [manifestDirFull, root, REPO_ROOT], + }); + } catch { + throw new Error( + `Cannot mechanically resolve Playwright test runner in ${manifestDir}: @playwright/test/cli could not be resolved. Topology validation must fail closed.` + ); + } + + let tempConfigPath = null; + let customConfigArg = null; + let overrideContent = null; + if (options.playwrightConfigOverrides) { + for (const [key, val] of Object.entries(options.playwrightConfigOverrides)) { + const normKey = key.replace(/^\.\//, '').split(sep).join('/'); + if ( + normKey === `${manifestDir}/playwright.config.ts` || + normKey === `${manifestDir}/playwright.config.js` || + normKey === `${manifestDir}/playwright.config.mjs` || + normKey === `${manifestDir}/playwright.config.cjs` + ) { + overrideContent = val; + break; + } + } + } + + if (overrideContent) { + const rand = randomBytes(6).toString('hex'); + const tempFileName = `.playwright.topology-temp-${rand}.ts`; + tempConfigPath = join(manifestDirFull, tempFileName); + writeFileSync(tempConfigPath, overrideContent, 'utf8'); + customConfigArg = `--config=${tempFileName}`; + } else if (options.scriptCmd) { + const configMatch = options.scriptCmd.match(/(?:--config|-c)[=\s]+(\S+)/); + if (configMatch) { + customConfigArg = `--config=${configMatch[1]}`; + } + } + + try { + const args = ['test', '--list', '--reporter=json']; + if (customConfigArg) { + args.push(customConfigArg); + } + + const res = spawnSync(process.execPath, [cliPath, ...args], { + cwd: manifestDirFull, + encoding: 'utf8', + env: { + ...process.env, + PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: '1', + // Validate reachability in the complete CI acceptance suite. The local + // backend-free mode intentionally discovers only offline specs. + GAMBIT_E2E_BACKEND: options.backend === false ? '' : '1', + }, + maxBuffer: 16 * 1024 * 1024, + }); + + if (res.error) { + throw new Error( + `Failed to execute Playwright discovery in ${manifestDir}: ${res.error.message}. Topology validation must fail closed.` + ); + } + + let parsed = null; + if (res.stdout && res.stdout.trim().startsWith('{')) { + try { + parsed = JSON.parse(res.stdout); + } catch { + parsed = null; + } + } + + const isZeroTestsFound = + parsed && + Array.isArray(parsed.suites) && + parsed.suites.length === 0 && + Array.isArray(parsed.errors) && + parsed.errors.every((e) => typeof e?.message === 'string' && e.message.includes('No tests found')); + + if (res.status !== 0 && !isZeroTestsFound) { + const errorMsg = (res.stderr || res.stdout || `process exited with status ${res.status}`).trim(); + throw new Error( + `Cannot mechanically resolve Playwright configuration in ${manifestDir}: ${errorMsg}. Topology validation must fail closed.` + ); + } + + if (!parsed || !Array.isArray(parsed.suites)) { + throw new Error( + `Cannot mechanically resolve Playwright configuration in ${manifestDir}: invalid or empty discovery payload. Topology validation must fail closed.` + ); + } + + const discovered = new Set(); + const configRootDir = parsed.config?.rootDir || manifestDirFull; + for (const suite of parsed.suites) { + collectPlaywrightFiles(suite, configRootDir, discovered, root); + } + + if (cache) { + cache.set(cacheKey, discovered); + } + + return discovered; + } finally { + if (tempConfigPath && existsSync(tempConfigPath)) { + try { + unlinkSync(tempConfigPath); + } catch { + // Ignore cleanup failure + } + } + } +} + +/** + * Mechanically resolves test patterns from a Playwright configuration file. + * Reads the actual Playwright config file from the workspace to extract `testDir` and `testMatch`, + * eliminating hard-coded duplicate reachability definitions. + * + * Fails closed: if `testDir` or `testMatch` is present but non-literal or unparseable, + * it explicitly throws an Error instead of substituting false-green defaults. + * + * @param {string} [manifestDir='packages/web'] - Directory containing playwright.config.* or package manifest. + * @param {object} [options={}] - Options containing root or config overrides. + * @returns {string[]} Resolved patterns relative to manifestDir. + */ +export function extractPlaywrightPatterns(manifestDir = 'packages/web', options = {}) { + const root = options.root || REPO_ROOT; + const configRel = join(manifestDir, 'playwright.config.ts').split(sep).join('/'); + let content = options.playwrightConfigOverrides?.[configRel]; + if (!content) { + const fullPath = join(root, configRel); + if (existsSync(fullPath)) { + content = readFileSync(fullPath, 'utf8'); + } + } + if (!content) { + for (const ext of ['.js', '.mjs', '.cjs']) { + const altRel = join(manifestDir, `playwright.config${ext}`).split(sep).join('/'); + if (options.playwrightConfigOverrides?.[altRel]) { + content = options.playwrightConfigOverrides[altRel]; + break; + } + const altFull = join(root, altRel); + if (existsSync(altFull)) { + content = readFileSync(altFull, 'utf8'); + break; + } + } + } + if (!content) { + return ['**/*.@(spec|test).ts']; + } + + const stripped = stripComments(content); + const objectBody = findExportedPlaywrightConfigBody(stripped, configRel); + const directProps = extractDirectProperties(objectBody); + + let testDir = null; + if (!directProps.has('testDir')) { + // Property absent -> Playwright default ('.' relative to manifestDir) is allowed + testDir = '.'; + } else { + const rawVal = directProps.get('testDir'); + const parsedLiteral = parseStaticStringLiteral(rawVal); + if (parsedLiteral === null) { + throw new Error( + `Cannot mechanically resolve Playwright 'testDir' in ${configRel}: property is present but non-literal or unparseable ("${rawVal}"). Topology validation must fail closed.` + ); + } + testDir = parsedLiteral.replace(/^\.\//, '').replace(/\/+$/, '') || '.'; + } + + let testMatchPatterns = null; + if (!directProps.has('testMatch')) { + // Property absent -> Playwright default ('**/*.spec.ts') is allowed + testMatchPatterns = ['**/*.spec.ts']; + } else { + const rawVal = directProps.get('testMatch'); + if (rawVal.startsWith('[')) { + const parsedArray = parseStringLiteralArray(rawVal); + if (parsedArray === null) { + throw new Error( + `Cannot mechanically resolve Playwright 'testMatch' in ${configRel}: property is present but contains non-literal or unparseable array elements. Topology validation must fail closed.` + ); + } + testMatchPatterns = parsedArray; + } else { + const parsedLiteral = parseStaticStringLiteral(rawVal); + if (parsedLiteral === null) { + throw new Error( + `Cannot mechanically resolve Playwright 'testMatch' in ${configRel}: property is present but non-literal or unsupported ("${rawVal}"). Topology validation must fail closed.` + ); + } + testMatchPatterns = [parsedLiteral]; + } + } + + return testMatchPatterns.map((matchPattern) => { + return testDir === '.' ? matchPattern : `${testDir}/${matchPattern}`; + }); +} + +/** + * Mechanically extracts test runner target file globs/paths from a package.json script command. + * Parses flags and options out of `node --test` or `playwright test` command invocations. + * + * @param {string} scriptCmd - Script command from package.json manifest. + * @param {object} [options={}] - Context options (e.g. manifestDir, root, playwrightConfigOverrides). + * @returns {string[]} Array of extracted target globs or file paths. + */ +export function extractRunnerPatterns(scriptCmd, options = {}) { + if (!scriptCmd) return []; + const parts = scriptCmd.split('&&').map((s) => s.trim()); + const testSubCmd = parts.find((s) => s.includes('node --test') || s.includes('playwright test')); + if (!testSubCmd) return []; + if (testSubCmd.includes('playwright test')) { + return extractPlaywrightPatterns(options.manifestDir || 'packages/web', options); + } + + const afterTest = testSubCmd.slice(testSubCmd.indexOf('node --test') + 'node --test'.length).trim(); + const tokenRegex = /(?:\"([^\"]+)\"|'([^']+)'|(\S+))/g; + const patterns = []; + let m; + while ((m = tokenRegex.exec(afterTest)) !== null) { + const token = m[1] || m[2] || m[3]; + if (token.startsWith('-')) continue; + patterns.push(token); + } + return patterns; +} + +/** + * Evaluates whether a candidate path matches a runner glob pattern. + * Uses native node:path posix.matchesGlob with a regex fallback. + * + * @param {string} pattern - Glob or file path pattern. + * @param {string} candidate - Relative candidate file path to match. + * @returns {boolean} + */ +export function matchRunnerPattern(pattern, candidate) { + const normPattern = pattern.replace(/\\/g, '/'); + const normCandidate = candidate.replace(/\\/g, '/'); + try { + if (typeof posix.matchesGlob === 'function') { + return posix.matchesGlob(normCandidate, normPattern); + } + } catch { + // Fall through to regex matcher + } + return matchRunnerPatternFallback(normPattern, normCandidate); +} + +function globSegmentToRegex(segment) { + const escapeLiteral = (value) => value.replace(/[.+^${}()|[\]\\]/g, '\\$&'); + const simple = (value) => escapeLiteral(value) + .replace(/\*/g, '[^/]*') + .replace(/\?/g, '[^/]'); + + const negative = segment.match(/^!\(([^)]+)\)(.*)$/); + if (negative) { + const suffix = simple(negative[2]); + const alternatives = negative[1].split('|').map(simple).join('|'); + return `(?!(?:${alternatives})${suffix}$)[^/]*${suffix}`; + } + return simple(segment); +} + +/** + * Portable fallback for the runner globs used by this repository. Supports `*`, `?`, + * globstar directory segments, and the negative extglob used by hermetic unit scripts. + */ +export function matchRunnerPatternFallback(pattern, candidate) { + const segments = pattern.replace(/\\/g, '/').split('/'); + let reStr = ''; + for (let index = 0; index < segments.length; index++) { + const segment = segments[index]; + if (index > 0 && segments[index - 1] !== '**') reStr += '/'; + if (segment === '**') { + reStr += index < segments.length - 1 ? '(?:[^/]+/)*' : '.*'; + continue; + } + reStr += globSegmentToRegex(segment); + } + return new RegExp(`^${reStr}$`).test(candidate.replace(/\\/g, '/')); +} + +/** + * Resolves the authoritative manifest for a suite and test file, and mechanically checks + * whether the actual runner glob in package.json reaches the test file. + * + * @param {object} suite - Suite definition. + * @param {string} relPath - Repository-relative POSIX file path. + * @param {object} [options={}] - Verification options (root, manifestCache, manifestOverrides). + * @returns {boolean} True if the test file is reached by the actual runner glob in package.json. + */ +export function isTestFileReachableByRunner(suite, relPath, options = {}) { + const root = options.root || REPO_ROOT; + const manifestCache = options.manifestCache || new Map(); + const manifestOverrides = options.manifestOverrides || {}; + + let manifestPath = suite.manifest; + if (!manifestPath) { + const match = relPath.match(/^packages\/([^/]+)\//); + if (!match) return false; + manifestPath = `packages/${match[1]}/package.json`; + } + + let baseManifest = manifestCache.get(manifestPath); + if (!baseManifest) { + const fullPath = join(root, manifestPath); + if (existsSync(fullPath)) { + baseManifest = JSON.parse(readFileSync(fullPath, 'utf8')); + manifestCache.set(manifestPath, baseManifest); + } + } + + let manifest = baseManifest; + if (manifestOverrides[manifestPath]) { + manifest = { + ...baseManifest, + ...manifestOverrides[manifestPath], + scripts: { + ...(baseManifest?.scripts || {}), + ...(manifestOverrides[manifestPath].scripts || {}), + }, + }; + } + if (!manifest) return false; + + const scriptCmd = manifest.scripts?.[suite.script]; + if (!scriptCmd) return false; + + const manifestDir = dirname(manifestPath); + + if (scriptCmd.includes('playwright test')) { + const discovered = getPlaywrightDiscoveredFiles(manifestDir, { + ...options, + scriptCmd, + }); + return discovered.has(relPath); + } + + const patterns = extractRunnerPatterns(scriptCmd, { + manifestDir, + root, + playwrightConfigOverrides: options.playwrightConfigOverrides, + }); + if (!patterns || patterns.length === 0) return false; + + const relToManifest = manifestDir === '.' ? relPath : relative(manifestDir, relPath).split(sep).join('/'); + const compiledPath = relToManifest.startsWith('test/') + ? 'dist-test/test/' + relToManifest.slice(5).replace(/\.ts$/, '.js') + : null; + + for (const pat of patterns) { + let normPat = pat; + let checkRelPath = relPath; + if (pat.startsWith('../') || pat.startsWith('./')) { + normPat = join(manifestDir, pat).split(sep).join('/'); + } + + if ( + matchRunnerPattern(normPat, relToManifest) || + (compiledPath && matchRunnerPattern(normPat, compiledPath)) || + matchRunnerPattern(normPat, checkRelPath) || + matchRunnerPattern(normPat, relPath) + ) { + return true; + } + } + + return false; +} + +const RAW_SUITE_DEFINITIONS = [ + { + name: 'acceptance-playwright', + pattern: /^packages\/web\/e2e\/.*\.spec\.ts$/, + target: 'npm run e2e (m6-acceptance)', + manifest: 'packages/web/package.json', + script: 'e2e', + }, + { + name: 'gateway-redis-integration', + pattern: /^services\/gateway\/test\/.*\.integration\.test\.ts$/, + target: 'npm test in services/gateway (gateway-service)', + manifest: 'services/gateway/package.json', + script: 'test', + }, + { + name: 'gateway-unit', + pattern: /^services\/gateway\/test\/.*\.test\.ts$/, + target: 'npm test in services/gateway (gateway-service)', + manifest: 'services/gateway/package.json', + script: 'test', + }, + { + name: 'gateway-trusted-edge', + pattern: /^scripts\/nginx-trusted-edge-acceptance\.mjs$/, + target: 'npm run test:trusted-edge in services/gateway (gateway-service)', + manifest: 'services/gateway/package.json', + script: 'test:trusted-edge', + }, + { + name: 'gateway-web-delivery', + pattern: /^scripts\/nginx-web-delivery-acceptance\.mjs$/, + target: 'npm run test:web-delivery in services/gateway (gateway-service)', + manifest: 'services/gateway/package.json', + script: 'test:web-delivery', + }, + { + name: 'persistence-postgres-integration', + pattern: /^packages\/persistence\/test\/.*\.integration\.test\.ts$/, + target: 'npm run test:integration:postgres -w @chess-platform/persistence', + manifest: 'packages/persistence/package.json', + script: 'test:integration:postgres', + }, + { + name: 'persistence-unit', + pattern: /^packages\/persistence\/test\/.*\.test\.ts$/, + target: 'npm test -w @chess-platform/persistence (build-test)', + manifest: 'packages/persistence/package.json', + script: 'test:unit', + }, + { + name: 'api-postgres-integration', + pattern: /^packages\/api\/test\/.*\.integration\.test\.ts$/, + target: 'npm run test:integration:postgres -w @chess-platform/api', + manifest: 'packages/api/package.json', + script: 'test:integration:postgres', + }, + { + name: 'api-engine-smoke', + pattern: /^packages\/api\/test\/(analysis-.*smoke|analysis-real-stack)\.test\.ts$/, + target: 'npm run test:analysis-smoke -w @chess-platform/api (analysis-smoke)', + manifest: 'packages/api/package.json', + script: 'test:analysis-smoke', + }, + { + name: 'api-diagnostics', + pattern: /^packages\/api\/test\/diagnostics\/.*\.diag\.ts$/, + target: 'npm run test:diagnostics:abort -w @chess-platform/api', + manifest: 'packages/api/package.json', + script: 'test:diagnostics:abort', + }, + { + name: 'api-posix-unit', + pattern: /^packages\/api\/test\/.*\.posix\.test\.ts$/, + target: 'npm run test:posix -w @chess-platform/api', + manifest: 'packages/api/package.json', + script: 'test:posix', + }, + { + name: 'api-unit', + pattern: /^packages\/api\/test\/.*\.test\.ts$/, + target: 'npm test -w @chess-platform/api (build-test)', + manifest: 'packages/api/package.json', + script: 'test:unit', + }, + { + name: 'ai-orchestrator-live-provider', + pattern: /^packages\/ai-orchestrator\/test\/.*\.integration\.test\.ts$/, + target: 'npm run test:live-provider -w @chess-platform/ai-orchestrator', + manifest: 'packages/ai-orchestrator/package.json', + script: 'test:live-provider', + }, + { + name: 'ai-orchestrator-unit', + pattern: /^packages\/ai-orchestrator\/test\/.*\.test\.ts$/, + target: 'npm test -w @chess-platform/ai-orchestrator (build-test)', + manifest: 'packages/ai-orchestrator/package.json', + script: 'test:unit', + }, + { + name: 'ai-features-live-provider', + pattern: /^packages\/ai-features\/test\/.*integration\.test\.ts$/, + target: 'npm run test:live-provider -w @chess-platform/ai-features', + manifest: 'packages/ai-features/package.json', + script: 'test:live-provider', + }, + { + name: 'ai-features-unit', + pattern: /^packages\/ai-features\/test\/.*\.test\.ts$/, + target: 'npm test -w @chess-platform/ai-features (build-test)', + manifest: 'packages/ai-features/package.json', + script: 'test:unit', + }, + { + name: 'scripts-postgres-integration', + pattern: /^scripts\/test\/.*\.integration\.test\.mjs$/, + target: 'npm run test:scripts:integration (postgres-integration)', + manifest: 'package.json', + script: 'test:scripts:integration', + }, + { + name: 'scripts-unit', + pattern: /^scripts\/test\/.*\.test\.mjs$/, + target: 'npm run test:scripts (build-test)', + manifest: 'package.json', + script: 'test:scripts', + }, + { + name: 'load-harness-posix', + pattern: /^deploy\/load\/test\/.*\.posix\.test\.mjs$/, + target: 'npm run test:load-harness:posix', + manifest: 'package.json', + script: 'test:load-harness:posix', + }, + { + name: 'load-harness-unit', + pattern: /^deploy\/load\/test\/.*\.test\.mjs$/, + target: 'npm run test:load-harness (build-test)', + manifest: 'package.json', + script: 'test:load-harness', + }, + { + name: 'domain-hermetic-unit', + pattern: /^packages\/[^/]+\/test\/.*\.test\.ts$/, + target: 'npm test (build-test)', + manifest: null, + script: 'test', + }, +]; + +export const SUITE_DEFINITIONS = RAW_SUITE_DEFINITIONS.map((suite) => ({ + ...suite, + isReachable(relPath, options) { + return isTestFileReachableByRunner(this, relPath, options); + }, +})); + +/** + * Validates whether a file path is located within an authorized test directory structure. + * + * @param {string} relPath - Repository-relative POSIX file path. + * @returns {boolean} True if the path resides in an approved test location. + */ +export function isAllowedPlacement(relPath) { + if ( + relPath === 'scripts/nginx-trusted-edge-acceptance.mjs' || + relPath === 'scripts/nginx-web-delivery-acceptance.mjs' + ) return true; + if (/^packages\/[^/]+\/test\/.+/.test(relPath)) return true; + if (/^packages\/web\/e2e\/.+/.test(relPath)) return true; + if (/^services\/[^/]+\/test\/.+/.test(relPath)) return true; + if (/^scripts\/test\/.+/.test(relPath)) return true; + if (/^deploy\/[^/]+\/test\/.+/.test(relPath)) return true; + return false; +} + +/** + * Recursively scans the repository from the given root directory to discover all test files, + * regardless of whether they reside in a /test/ directory or are misplaced in src/. + * Excludes build artifacts, dependencies, and generated reports. + * + * @param {string} [root=REPO_ROOT] - Repository root directory to scan. + * @returns {string[]} Sorted array of repository-relative POSIX file paths for all discovered tests. + */ +export function findTestFiles(root = REPO_ROOT) { + const testFiles = []; + + function walk(dir) { + const entries = readdirSync(dir, { withFileTypes: true }); + for (const entry of entries) { + const fullPath = join(dir, entry.name); + const relPath = relative(root, fullPath).split(sep).join('/'); + + if (entry.isDirectory()) { + if ( + entry.name === 'node_modules' || + entry.name === 'dist' || + entry.name === 'dist-test' || + entry.name === '.git' || + entry.name === 'coverage' || + entry.name === 'playwright-report' || + entry.name === 'test-results' || + relPath === 'deploy/helm' || + relPath === 'deploy/observability' + ) { + continue; + } + walk(fullPath); + } else if (entry.isFile()) { + if ( + relPath === 'scripts/nginx-trusted-edge-acceptance.mjs' || + relPath === 'scripts/nginx-web-delivery-acceptance.mjs' || + /\.(test|spec|diag)\.(ts|js|mjs|cjs)$/.test(relPath) + ) { + testFiles.push(relPath); + } + } + } + } + + walk(root); + return testFiles.sort(); +} + +/** + * Matches a repository-relative test file path against known zero-skip suite definitions. + * + * @param {string} relPath - Repository-relative file path in POSIX format. + * @returns {object|null} The matching suite definition object, or null if unclassified. + */ +export function classifyTestFile(relPath) { + for (const suite of SUITE_DEFINITIONS) { + if (suite.pattern.test(relPath)) { + return suite; + } + } + return null; +} + +/** + * Scans the repository and validates that: + * 1. 100% of test files are placed in authorized test directories (no src/ co-location). + * 2. 100% of test files map to an explicit zero-skip suite. + * 3. 100% of test files are reachable by their owning suite's runner execution globs. + * + * @param {string} [root=REPO_ROOT] - Repository root directory to verify. + * @returns {{ + * totalFiles: number, + * misplaced: string[], + * unclassified: string[], + * unreachable: Array<{ file: string, suite: string }>, + * categorized: Map + * }} + */ +export function verifyTestTopology(root = REPO_ROOT, options = {}) { + const files = findTestFiles(root); + const misplaced = []; + const unclassified = []; + const unreachable = []; + const categorized = new Map(); + const manifestCache = new Map(); + const playwrightDiscoveredCache = options.playwrightDiscoveredCache || new Map(); + + for (const file of files) { + if (!isAllowedPlacement(file)) { + misplaced.push(file); + continue; + } + + const suite = classifyTestFile(file); + if (!suite) { + unclassified.push(file); + continue; + } + + if (suite.isReachable && !suite.isReachable(file, { root, manifestCache, playwrightDiscoveredCache, ...options })) { + unreachable.push({ file, suite: suite.name }); + continue; + } + + const list = categorized.get(suite.name) || []; + list.push(file); + categorized.set(suite.name, list); + } + + return { + totalFiles: files.length, + misplaced, + unclassified, + unreachable, + categorized, + }; +} + +if (process.argv[1] && resolve(process.argv[1]) === resolve(import.meta.filename)) { + let result; + try { + result = verifyTestTopology(); + } catch (err) { + console.error(`[TEST TOPOLOGY] FAILED: ${err.message}`); + process.exit(1); + } + console.log(`[TEST TOPOLOGY] Verified ${result.totalFiles} test files across ${result.categorized.size} suites.`); + + for (const [suiteName, files] of result.categorized.entries()) { + console.log(` - ${suiteName}: ${files.length} file(s)`); + } + + let failed = false; + + if (result.misplaced.length > 0) { + console.error(`[TEST TOPOLOGY] FAILED: ${result.misplaced.length} misplaced test file(s) found (outside authorized test directories):`); + for (const f of result.misplaced) { + console.error(` - ${f}`); + } + failed = true; + } + + if (result.unclassified.length > 0) { + console.error(`[TEST TOPOLOGY] FAILED: ${result.unclassified.length} unclassified test file(s) found:`); + for (const f of result.unclassified) { + console.error(` - ${f}`); + } + failed = true; + } + + if (result.unreachable.length > 0) { + console.error(`[TEST TOPOLOGY] FAILED: ${result.unreachable.length} test file(s) unreachable by suite runner:`); + for (const item of result.unreachable) { + console.error(` - ${item.file} (suite: ${item.suite})`); + } + failed = true; + } + + if (failed) { + process.exit(1); + } + + console.log('[TEST TOPOLOGY] All test files successfully placed, classified, and reachable by explicit zero-skip suites.'); +} + diff --git a/scripts/ci-local.mjs b/scripts/ci-local.mjs index fa79445e..faba3209 100644 --- a/scripts/ci-local.mjs +++ b/scripts/ci-local.mjs @@ -38,6 +38,7 @@ const CORE = [ ['CI parity', 'npm run check:ci-parity'], ['variant parity', 'npm run check:variant-parity'], ['engine pin parity', 'npm run check:engine-pin-parity'], + ['test topology', 'npm run check:test-topology'], ['guard script tests', 'npm run test:scripts'], ]; @@ -79,8 +80,9 @@ const SERVICE_JOBS = [ ? 'DATABASE_URL does not name a database with "test" in it — these suites need a disposable, empty one' : null, steps: [ - ['persistence against Postgres', 'npm test --workspace @chess-platform/persistence'], - ['api concurrency against Postgres', 'npm test --workspace @chess-platform/api'], + ['persistence against Postgres', 'npm run test:integration:postgres --workspace @chess-platform/persistence'], + ['api concurrency against Postgres', 'npm run test:integration:postgres --workspace @chess-platform/api'], + ['scripts integration against Postgres', 'npm run test:scripts:integration'], ], }, { @@ -107,6 +109,15 @@ const SERVICE_JOBS = [ ['web delivery caching and compression through real Nginx', 'npm run test:web-delivery'], ], }, + { + name: 'posix contract tests (Linux/macOS only)', + needs: 'POSIX platform', + available: process.platform !== 'win32', + steps: [ + ['POSIX API tests', 'npm run test:posix -w @chess-platform/api'], + ['POSIX load harness tests', 'npm run test:load-harness:posix'], + ], + }, ]; /** diff --git a/scripts/db-backup-restore-drill.mjs b/scripts/db-backup-restore-drill.mjs index bd03e0a5..2368471c 100644 --- a/scripts/db-backup-restore-drill.mjs +++ b/scripts/db-backup-restore-drill.mjs @@ -801,7 +801,10 @@ export async function runBackupRestoreDrill(options = {}) { }); const sourcePool = new Pool({ connectionString: options.sourceUrl, max: 2 }); + sourcePool.on('connect', (client) => { client.on('error', () => {}); }); + sourcePool.on('error', () => {}); let targetPool = null; + const targetClients = new Set(); let adminClient = null; let targetCreatedByThisRun = false; let backupCreatedByThisRun = false; @@ -919,11 +922,13 @@ export async function runBackupRestoreDrill(options = {}) { log(`Provisioning isolated target database "${parsedTarget.database}"...`); const adminUrl = urlWithDatabase(targetUrl, 'postgres'); adminClient = new Client({ connectionString: adminUrl, statement_timeout: 10000 }); + adminClient.on('error', () => {}); try { await adminClient.connect(); } catch { // Try template1 if postgres db is not accessible adminClient = new Client({ connectionString: urlWithDatabase(targetUrl, 'template1'), statement_timeout: 10000 }); + adminClient.on('error', () => {}); await adminClient.connect(); } @@ -1009,6 +1014,14 @@ export async function runBackupRestoreDrill(options = {}) { log('Running comprehensive structural and functional verification...'); const verifyStart = Date.now(); targetPool = new Pool({ connectionString: targetUrl, max: 2 }); + targetPool.on('connect', (client) => { + targetClients.add(client); + client.on('error', () => {}); + }); + targetPool.on('remove', (client) => { + targetClients.delete(client); + }); + targetPool.on('error', () => {}); const verifyResult = await verifyRestoredDatabase(sourceBaseline, targetPool, options); report.timings.verifyMs = Date.now() - verifyStart; report.checks = verifyResult.checks; @@ -1021,6 +1034,10 @@ export async function runBackupRestoreDrill(options = {}) { // Teardown connections await sourcePool.end().catch(err => cleanupErrors.push(err)); if (targetPool) { + targetPool.on('error', () => {}); + for (const client of targetClients) { + client.on('error', () => {}); + } await targetPool.end().catch(err => cleanupErrors.push(err)); } @@ -1029,6 +1046,9 @@ export async function runBackupRestoreDrill(options = {}) { if (!options.keepTarget && parsedTarget.database && targetCreatedByThisRun) { try { log(`Cleaning up isolated target database "${parsedTarget.database}"...`); + for (const client of targetClients) { + client.on('error', () => {}); + } await adminClient.query(`DROP DATABASE IF EXISTS "` + parsedTarget.database.replace(/"/g, '""') + `" WITH (FORCE)`); log('Target database dropped.'); } catch (err) { diff --git a/scripts/lib/test-output-parser.mjs b/scripts/lib/test-output-parser.mjs new file mode 100644 index 00000000..c29e352f --- /dev/null +++ b/scripts/lib/test-output-parser.mjs @@ -0,0 +1,525 @@ +/** + * @file Reusable parsing helpers for test runner output metrics, directives, and plans. + * Supports TAP and Node.js spec reporter formats, strictly enforcing zero-skip, + * zero-todo, and zero-cancelled test quality gates. + */ +import { StringDecoder } from 'node:string_decoder'; + +const ANSI_ESCAPE_REGEX = /\x1b\[[0-?]*[ -/]*[@-~]/g; + +/** + * Removes terminal color/control sequences before matching line-anchored reporter records. + * + * @param {string} value - Raw reporter output. + * @returns {string} Output without ANSI CSI escape sequences. + */ +export function stripAnsi(value) { + return value.replace(ANSI_ESCAPE_REGEX, ''); +} + +/** + * Line-anchored regex matching individual TAP or spec skip directives. + * TAP: "ok 1 - test # SKIP [reason]", "ok - test # SKIP [reason]", or "not ok ... # SKIP" + * Spec: "- test # SKIP [reason]" or "- test (skipped)" + * Ensures escaped hashes (`\#`) in descriptions are not misinterpreted as directives. + */ +export const TAP_OR_SPEC_SKIP_DIRECTIVE_REGEX = + /^[ \t]*(?:(?:ok|not ok)(?=[ \t]|$)[^\r\n]*?(? 0) { + return summaryMatches.reduce((sum, m) => sum + Number.parseInt(m[1], 10), 0); + } + if (topLevelPlanMatches.length === 1) { + return Number.parseInt(topLevelPlanMatches[0][1], 10); + } + return null; +} + +/** + * Resolves the passing test count from reporter summary lines. + * + * @param {string} output - Combined stdout/stderr text output. + * @returns {number|null} Aggregated passing count, or null if no pass summary lines exist. + */ +export function parsePassCount(output) { + output = stripAnsi(output); + const matches = [...output.matchAll(/^\s*(?:#|ℹ)\s+pass(?:ed)?:?\s+(\d+)\b/gim)]; + if (matches.length === 0) return null; + return matches.reduce((sum, m) => sum + Number.parseInt(m[1], 10), 0); +} + +/** + * Resolves the failure test count from reporter summary lines and raw TAP "not ok" records. + * + * @param {string} output - Combined stdout/stderr text output. + * @returns {number} Aggregated failure count across summaries and raw TAP records. + */ +export function parseFailCount(output) { + output = stripAnsi(output); + let count = 0; + const matches = [...output.matchAll(/^\s*(?:#|ℹ)\s+fail(?:ed)?:?\s+(\d+)\b/gim)]; + for (const match of matches) { + count += Number.parseInt(match[1], 10); + } + if (RAW_TAP_FAILURE_REGEX.test(output) && count === 0) { + count = 1; + } + return count; +} + +/** + * Resolves the skipped test count from reporter summary lines and individual test skip directives. + * Independently detects record-level # SKIP directives even if summary reports 0. + * + * @param {string} output - Combined stdout/stderr text output. + * @returns {number} Aggregated skipped count across summaries and directives. + */ +export function parseSkippedCount(output) { + output = stripAnsi(output); + let count = 0; + const matches = [...output.matchAll(/^\s*(?:#|ℹ)\s+skipped:?\s+(\d+)\b/gim)]; + for (const match of matches) { + count += Number.parseInt(match[1], 10); + } + if (TAP_OR_SPEC_SKIP_DIRECTIVE_REGEX.test(output) && count === 0) { + count = 1; + } + return count; +} + +/** + * Resolves the TODO test count from reporter summary lines and individual test TODO directives. + * Independently detects record-level # TODO directives even if summary reports 0. + * + * @param {string} output - Combined stdout/stderr text output. + * @returns {number} Aggregated TODO count across summaries and directives. + */ +export function parseTodoCount(output) { + output = stripAnsi(output); + let count = 0; + const matches = [...output.matchAll(/^\s*(?:#|ℹ)\s+todo:?\s+(\d+)\b/gim)]; + for (const match of matches) { + count += Number.parseInt(match[1], 10); + } + if (TAP_OR_SPEC_TODO_DIRECTIVE_REGEX.test(output) && count === 0) { + count = 1; + } + return count; +} + +/** + * Resolves the cancelled test count from reporter summary lines and spec cancelled format. + * + * @param {string} output - Combined stdout/stderr text output. + * @returns {number} Aggregated cancelled count. + */ +export function parseCancelledCount(output) { + output = stripAnsi(output); + let count = 0; + const matches = [...output.matchAll(/^\s*(?:#|ℹ)\s+cancelled:?\s+(\d+)\b/gim)]; + for (const match of matches) { + count += Number.parseInt(match[1], 10); + } + const specCancelledRegex = /^\s*[\ufe63\-]\s+[^\r\n]*\((?:cancelled)\)/im; + if (specCancelledRegex.test(output) && count === 0) { + count = 1; + } + return count; +} + +/** + * Creates a stateful streaming test runner output parser that maintains strictly O(1) + * bounded memory by updating scalar counters and flags line-by-line as chunks arrive. + * Eliminates transcript accumulation while guaranteeing exact parity with static summary + * and directive quality gates (zero skip, zero todo, zero cancelled, zero unhandled failures). + * + * @returns {{ + * pushLine: (line: string) => void, + * getResults: () => { + * totalTests: number | null, + * passCount: number | null, + * failCount: number, + * skippedCount: number, + * todoCount: number, + * cancelledCount: number + * } + * }} + */ +export function createStreamingTestParser() { + let summaryTests = 0; + let summaryTestsCount = 0; + let hasSpecTestSummary = false; + let hasTapTestSummary = false; + let planTestsCount = 0; + let hasZeroTestSummary = false; + + let summaryPass = 0; + let summaryPassCount = 0; + + let summarySkipped = 0; + let hasSkipDirective = false; + + let summaryTodo = 0; + let hasTodoDirective = false; + + let summaryCancelled = 0; + let hasCancelledDirective = false; + + let summaryFail = 0; + let hasRawFailure = false; + + const summaryMetricNames = ['tests', 'pass', 'fail', 'skipped', 'todo', 'cancelled']; + let currentSummary = null; + let completedSummaryCount = 0; + let topLevelPlanCount = 0; + let topLevelPlanValue = null; + let topLevelTapPoints = 0; + let topLevelPassPoints = 0; + let malformedSummary = null; + + const metricPrefixRegex = /^\s*(?:#|ℹ)\s+(tests|pass(?:ed)?|fail(?:ed)?|skipped|todo|cancelled)(?::|\s*$|\s+[-+\d])/i; + const numericMetrics = { + tests: /^\s*(?:#|ℹ)\s+tests:?\s+(\d+)\s*$/i, + pass: /^\s*(?:#|ℹ)\s+pass(?:ed)?:?\s+(\d+)\s*$/i, + fail: /^\s*(?:#|ℹ)\s+fail(?:ed)?:?\s+(\d+)\s*$/i, + skipped: /^\s*(?:#|ℹ)\s+skipped:?\s+(\d+)\s*$/i, + todo: /^\s*(?:#|ℹ)\s+todo:?\s+(\d+)\s*$/i, + cancelled: /^\s*(?:#|ℹ)\s+cancelled:?\s+(\d+)\s*$/i, + }; + + function recordSummaryMetric(metric, value) { + if (metric === 'tests') { + if (currentSummary !== null) { + const missing = summaryMetricNames.filter((name) => currentSummary[name] === null); + malformedSummary ??= `Incomplete reporter summary before next tests field: missing ${missing.join(', ')}`; + } + currentSummary = { + tests: value, + pass: null, + fail: null, + skipped: null, + todo: null, + cancelled: null, + }; + return; + } + + if (currentSummary === null) { + malformedSummary ??= `Reporter ${metric} field appeared without a preceding tests field`; + return; + } + + if (currentSummary[metric] !== null) { + malformedSummary ??= `Duplicate ${metric} field in reporter summary`; + return; + } + + currentSummary[metric] = value; + if (summaryMetricNames.some((name) => currentSummary[name] === null)) return; + + const accounted = currentSummary.pass + + currentSummary.fail + + currentSummary.skipped + + currentSummary.todo + + currentSummary.cancelled; + if (accounted !== currentSummary.tests) { + malformedSummary ??= `Contradictory reporter summary: tests=${currentSummary.tests} but pass+fail+skipped+todo+cancelled=${accounted}`; + } + completedSummaryCount++; + currentSummary = null; + } + + return { + pushLine(line) { + line = stripAnsi(line).replace(/\r$/, ''); + + const summaryLike = line.match(metricPrefixRegex); + if (summaryLike && !/^\s*(?:#|ℹ)\s+(?:tests|pass(?:ed)?|fail(?:ed)?|skipped|todo|cancelled):?\s+\d+\s*$/i.test(line)) { + malformedSummary ??= `Malformed ${summaryLike[1]} summary line: ${line}`; + } + + // 1. Tests summary: # tests N or ℹ tests N + const testMatch = line.match(numericMetrics.tests); + if (testMatch) { + const val = Number.parseInt(testMatch[1], 10); + if (val === 0) hasZeroTestSummary = true; + if (/^\s*ℹ/.test(line)) hasSpecTestSummary = true; + else hasTapTestSummary = true; + summaryTests += val; + summaryTestsCount++; + recordSummaryMetric('tests', val); + } + + // 2. TAP plan: 1..N + const planMatch = line.match(/^[ \t]*1\.\.(\d+)(?:[ \t]*#.*)?[ \t]*$/); + if (planMatch) { + const val = Number.parseInt(planMatch[1], 10); + if (val === 0) hasZeroTestSummary = true; + planTestsCount++; + if (/^1\.\./.test(line)) { + topLevelPlanCount++; + topLevelPlanValue = val; + } + } + + const tapPoint = line.match(/^(ok|not ok)(?=[ \t]|$)/i); + if (tapPoint) { + topLevelTapPoints++; + if (tapPoint[1].toLowerCase() === 'ok' + && !TAP_OR_SPEC_SKIP_DIRECTIVE_REGEX.test(line) + && !TAP_OR_SPEC_TODO_DIRECTIVE_REGEX.test(line)) { + topLevelPassPoints++; + } + } + + // 3. Pass summary: # pass N or ℹ pass N + const passMatch = line.match(numericMetrics.pass); + if (passMatch) { + const val = Number.parseInt(passMatch[1], 10); + summaryPass += val; + summaryPassCount++; + recordSummaryMetric('pass', val); + } + + // 4. Skipped summary or directive: # skipped N or ℹ skipped N or TAP/spec skip + const skipMatch = line.match(numericMetrics.skipped); + if (skipMatch) { + const val = Number.parseInt(skipMatch[1], 10); + summarySkipped += val; + recordSummaryMetric('skipped', val); + } else if (TAP_OR_SPEC_SKIP_DIRECTIVE_REGEX.test(line)) { + hasSkipDirective = true; + } + + // 5. TODO summary or directive: # todo N or ℹ todo N or TAP/spec todo + const todoMatch = line.match(numericMetrics.todo); + if (todoMatch) { + const val = Number.parseInt(todoMatch[1], 10); + summaryTodo += val; + recordSummaryMetric('todo', val); + } else if (TAP_OR_SPEC_TODO_DIRECTIVE_REGEX.test(line)) { + hasTodoDirective = true; + } + + // 6. Cancelled summary or directive: # cancelled N or spec (cancelled) + const cancelledMatch = line.match(numericMetrics.cancelled); + if (cancelledMatch) { + const val = Number.parseInt(cancelledMatch[1], 10); + summaryCancelled += val; + recordSummaryMetric('cancelled', val); + } else if (/^\s*[\ufe63\-]\s+[^\r\n]*\((?:cancelled)\)/i.test(line)) { + hasCancelledDirective = true; + } + + // 7. Fail summary or raw TAP "not ok": # fail N or not ok + const failMatch = line.match(numericMetrics.fail); + if (failMatch) { + const val = Number.parseInt(failMatch[1], 10); + summaryFail += val; + recordSummaryMetric('fail', val); + } else if (RAW_TAP_FAILURE_REGEX.test(line)) { + hasRawFailure = true; + } + }, + + getResults() { + let totalTests = null; + if (hasZeroTestSummary) { + totalTests = 0; + } else if (summaryTestsCount > 0) { + totalTests = summaryTests; + } else if (topLevelPlanCount === 1) { + totalTests = topLevelPlanValue; + } + + let passCount = summaryPassCount > 0 ? summaryPass : null; + + let accountingError = malformedSummary; + if (!accountingError && summaryTestsCount > 0) { + if (currentSummary !== null) { + const missing = summaryMetricNames.filter((name) => currentSummary[name] === null); + accountingError = `Incomplete reporter summary: missing ${missing.join(', ')}`; + } else if (completedSummaryCount !== summaryTestsCount) { + accountingError = `Incomplete reporter summaries: completed ${completedSummaryCount} of ${summaryTestsCount}`; + } else { + const accounted = summaryPass + summaryFail + summarySkipped + summaryTodo + summaryCancelled; + if (accounted !== summaryTests) { + accountingError = `Contradictory reporter summaries: tests=${summaryTests} but pass+fail+skipped+todo+cancelled=${accounted}`; + } + } + } + + // A reporter summary does not excuse missing or contradictory top-level TAP evidence. + // Indented subtest plans describe a different scope and cannot be added to this plan. + // Spec reporters use the ℹ summary and can include application logs beginning + // with TAP-like words. TAP summaries and plan-only streams remain fail-closed. + const specOnlySummary = hasSpecTestSummary && !hasTapTestSummary; + const requiresTapReconciliation = planTestsCount > 0 + || (topLevelTapPoints > 0 && !specOnlySummary); + if (!accountingError && requiresTapReconciliation) { + if (topLevelPlanCount !== 1 || topLevelPlanValue === null) { + accountingError = `Incomplete TAP evidence: expected exactly one top-level plan, found ${topLevelPlanCount}`; + } else if (topLevelTapPoints !== topLevelPlanValue) { + accountingError = `Contradictory TAP evidence: top-level plan declares ${topLevelPlanValue} test point(s) but ${topLevelTapPoints} top-level point(s) were observed`; + } else if (summaryTestsCount === 0) { + passCount = topLevelPassPoints; + } + } + + let skippedCount = summarySkipped; + if (skippedCount === 0 && hasSkipDirective) skippedCount = 1; + + let todoCount = summaryTodo; + if (todoCount === 0 && hasTodoDirective) todoCount = 1; + + let cancelledCount = summaryCancelled; + if (cancelledCount === 0 && hasCancelledDirective) cancelledCount = 1; + + let failCount = summaryFail; + if (failCount === 0 && hasRawFailure) failCount = 1; + + return { + totalTests, + passCount, + failCount, + skippedCount, + todoCount, + cancelledCount, + accountingValid: accountingError === null, + accountingError, + }; + }, + reportMalformedOutput(reason) { + malformedSummary ??= reason; + }, + }; +} + +/** + * Parses a completed in-memory transcript with the same strict accounting rules used by the + * streaming zero-skip runner. + * + * @param {string} output - Complete stdout/stderr transcript. + * @returns {ReturnType['getResults']>} + */ +export function parseCompleteTestOutput(output) { + const parser = createStreamingTestParser(); + for (const line of output.split(/\r?\n/)) { + if (line.length > 0) parser.pushLine(line); + } + return parser.getResults(); +} + +/** + * Wraps a parser or test receiver to process streaming binary or string chunks from stdout + * and stderr independently. + * + * Maintains separate UTF-8 StringDecoder instances and line buffers for stdout and stderr, + * preventing cross-stream line interleaving corruption and multibyte sequence splitting. + * + * @param {ReturnType} parser + * @returns {{ + * pushStdoutChunk: (chunk: Buffer|string) => void, + * pushStderrChunk: (chunk: Buffer|string) => void, + * flush: () => void, + * getResults: () => ReturnType + * }} + */ +export function createStreamLineProcessor(parser) { + const MAX_TEST_OUTPUT_LINE_LENGTH = 1024 * 1024; + const stdoutDecoder = new StringDecoder('utf8'); + const stderrDecoder = new StringDecoder('utf8'); + let stdoutLineBuffer = ''; + let stderrLineBuffer = ''; + + function processChunk(chunk, decoder, getBuffer, setBuffer) { + const text = typeof chunk === 'string' ? chunk : decoder.write(chunk); + const combined = getBuffer() + text; + const lines = combined.split(/\r?\n/); + const pending = lines.pop() ?? ''; + if (pending.length > MAX_TEST_OUTPUT_LINE_LENGTH) { + parser.reportMalformedOutput(`Test output line exceeds ${MAX_TEST_OUTPUT_LINE_LENGTH} characters`); + setBuffer(''); + } else { + setBuffer(pending); + } + for (const line of lines) { + if (line.length > MAX_TEST_OUTPUT_LINE_LENGTH) { + parser.reportMalformedOutput(`Test output line exceeds ${MAX_TEST_OUTPUT_LINE_LENGTH} characters`); + } else { + parser.pushLine(line); + } + } + } + + function flushStream(decoder, getBuffer, setBuffer) { + const finalStr = decoder.end(); + const combined = getBuffer() + finalStr; + const lines = combined.split(/\r?\n/); + setBuffer(''); + for (const line of lines) { + if (line.length > 0) { + if (line.length > MAX_TEST_OUTPUT_LINE_LENGTH) { + parser.reportMalformedOutput(`Test output line exceeds ${MAX_TEST_OUTPUT_LINE_LENGTH} characters`); + } else { + parser.pushLine(line); + } + } + } + } + + return { + pushStdoutChunk(chunk) { + processChunk(chunk, stdoutDecoder, () => stdoutLineBuffer, (v) => { stdoutLineBuffer = v; }); + }, + pushStderrChunk(chunk) { + processChunk(chunk, stderrDecoder, () => stderrLineBuffer, (v) => { stderrLineBuffer = v; }); + }, + flush() { + flushStream(stdoutDecoder, () => stdoutLineBuffer, (v) => { stdoutLineBuffer = v; }); + flushStream(stderrDecoder, () => stderrLineBuffer, (v) => { stderrLineBuffer = v; }); + }, + getResults() { + this.flush(); + return parser.getResults(); + }, + }; +} + diff --git a/scripts/lib/workspace-topology.mjs b/scripts/lib/workspace-topology.mjs new file mode 100644 index 00000000..d04413ae --- /dev/null +++ b/scripts/lib/workspace-topology.mjs @@ -0,0 +1,99 @@ +/** + * @file Single authoritative source of workspace discovery and test topology metadata. + * Dynamically derives workspace packages from root package.json, preventing drift + * between hermetic runners, test count auditors, and topology checkers. + */ +import { existsSync, readdirSync, readFileSync } from 'node:fs'; +import { dirname, isAbsolute, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + + +const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); +export const REPO_ROOT = resolve(SCRIPT_DIR, '../..'); + +/** + * Reads root package.json and discovers all valid workspace package manifests. + * + * @param {string} [root=REPO_ROOT] - Repository root directory. + * @returns {Array<{ name: string, dir: string, relDir: string, manifest: object }>} + */ +export function discoverWorkspacePackages(root = REPO_ROOT) { + const rootPkgPath = join(root, 'package.json'); + if (!existsSync(rootPkgPath)) { + throw new Error(`Root package.json not found at ${rootPkgPath}`); + } + const rootPkg = JSON.parse(readFileSync(rootPkgPath, 'utf8')); + const workspacePatterns = rootPkg.workspaces || []; + + const packages = []; + + for (const pattern of workspacePatterns) { + if (pattern.endsWith('/*')) { + const baseDir = join(root, pattern.slice(0, -2)); + if (!existsSync(baseDir)) continue; + const entries = readdirSync(baseDir, { withFileTypes: true }); + for (const entry of entries) { + if (!entry.isDirectory()) continue; + const pkgDir = join(baseDir, entry.name); + const pkgJsonPath = join(pkgDir, 'package.json'); + if (existsSync(pkgJsonPath)) { + const manifest = JSON.parse(readFileSync(pkgJsonPath, 'utf8')); + packages.push({ + name: manifest.name, + dir: pkgDir, + relDir: relative(root, pkgDir).replace(/\\/g, '/'), + manifest, + }); + } + } + } else { + const pkgDir = join(root, pattern); + const pkgJsonPath = join(pkgDir, 'package.json'); + if (existsSync(pkgJsonPath)) { + const manifest = JSON.parse(readFileSync(pkgJsonPath, 'utf8')); + packages.push({ + name: manifest.name, + dir: pkgDir, + relDir: relative(root, pkgDir).replace(/\\/g, '/'), + manifest, + }); + } + } + } + + return packages.sort((a, b) => a.name.localeCompare(b.name)); +} + +/** + * Packages with partitioned non-hermetic or specialized execution suites. + * In these packages, `npm test` runs only the hermetic unit tests, while + * environment-dependent tests (Postgres, live providers, smoke) are partitioned. + */ +export const PARTITIONED_PACKAGES = Object.freeze([ + '@chess-platform/api', + '@chess-platform/persistence', + '@chess-platform/ai-orchestrator', + '@chess-platform/ai-features', +]); + +/** + * Derives the list of all hermetic package names to be executed during root `npm test`. + * Every workspace under `packages/` is hermetically tested during root `npm test`. + * + * @param {string} [root=REPO_ROOT] + * @returns {readonly string[]} Sorted list of workspace package names. + */ +export function getHermeticWorkspaces(root = REPO_ROOT) { + const packagesDir = resolve(root, 'packages'); + const discovered = discoverWorkspacePackages(root); + return Object.freeze( + discovered + .filter((pkg) => { + const resolvedPkgDir = resolve(pkg.dir); + const rel = relative(packagesDir, resolvedPkgDir); + return !rel.startsWith('..') && !isAbsolute(rel) && rel !== ''; + }) + .map((pkg) => pkg.name) + ); +} + diff --git a/scripts/playwright-zero-skip-reporter.mjs b/scripts/playwright-zero-skip-reporter.mjs new file mode 100644 index 00000000..b96b1773 --- /dev/null +++ b/scripts/playwright-zero-skip-reporter.mjs @@ -0,0 +1,73 @@ +/** + * Playwright reporter that makes the repository's zero-skip contract enforceable for browser + * suites. Playwright's process exit status alone permits annotations such as `test.skip()`. + */ + +/** + * @param {string[]} outcomes Playwright TestCase.outcome() values. + * @param {string} runStatus Playwright FullResult.status. + */ +export function summarizePlaywrightOutcomes(outcomes, runStatus) { + const summary = { + tests: outcomes.length, + pass: 0, + fail: 0, + skipped: 0, + todo: 0, + cancelled: 0, + }; + + for (const outcome of outcomes) { + if (outcome === 'skipped') summary.skipped++; + else if (outcome === 'unexpected') summary.fail++; + else if (outcome === 'expected' || outcome === 'flaky') summary.pass++; + else summary.cancelled++; + } + + // Preserve exact accounting while making a non-passing run visible in the summary. Playwright + // can report every test as expected even when teardown later times out or is interrupted. + if (runStatus !== 'passed' && summary.tests > 0 && summary.pass === summary.tests) { + summary.pass--; + if (runStatus === 'interrupted') summary.cancelled++; + else summary.fail++; + } + + return summary; +} + +export default class ZeroSkipReporter { + constructor() { + this.suite = null; + this.listOnly = process.argv.includes('--list'); + } + + onBegin(_config, suite) { + this.suite = suite; + } + + onEnd(result) { + if (this.listOnly) return { status: result.status }; + const outcomes = this.suite?.allTests().map((testCase) => testCase.outcome()) ?? []; + const summary = summarizePlaywrightOutcomes(outcomes, result.status); + + console.log(`# tests ${summary.tests}`); + console.log(`# pass ${summary.pass}`); + console.log(`# fail ${summary.fail}`); + console.log(`# cancelled ${summary.cancelled}`); + console.log(`# skipped ${summary.skipped}`); + console.log(`# todo ${summary.todo}`); + + const clean = + result.status === 'passed' && + summary.tests > 0 && + summary.pass === summary.tests && + summary.fail === 0 && + summary.cancelled === 0 && + summary.skipped === 0; + return { status: clean ? 'passed' : 'failed' }; + } + + printsToStdio() { + return true; + } +} diff --git a/scripts/run-hermetic-tests.mjs b/scripts/run-hermetic-tests.mjs new file mode 100644 index 00000000..a6401cf1 --- /dev/null +++ b/scripts/run-hermetic-tests.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/** + * Centralized zero-skip orchestrator for repository-wide hermetic workspace test fan-out. + * + * Runs each declared hermetic workspace in sequence through runWithZeroSkip, strictly + * enforcing that every child workspace executes tests, exits with code 0, and reports + * zero skipped tests. If any workspace reports skipped > 0, 0 tests, non-zero exit, + * or signal termination, execution halts immediately with failure. + */ +import { existsSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { getHermeticWorkspaces } from './lib/workspace-topology.mjs'; +import { runWithZeroSkip } from './run-zero-skip.mjs'; + +/** + * Derived list of all hermetic packages executed during root `npm test`. + * These packages contain hermetic unit tests with zero external service dependencies. + * + * @type {readonly string[]} + */ +export const HERMETIC_WORKSPACES = getHermeticWorkspaces(); + +/** + * Resolves the path to the npm-cli.js executable without relying on shell expansion. + * Checks process.env.npm_execpath first, then falls back to standard Node.js installation paths. + * + * @returns {string|null} Absolute path to npm-cli.js, or null if unresolvable. + */ +export function resolveNpmCli() { + if (process.env.npm_execpath && existsSync(process.env.npm_execpath)) { + return process.env.npm_execpath; + } + const winCandidate = join(dirname(process.execPath), 'node_modules', 'npm', 'bin', 'npm-cli.js'); + if (existsSync(winCandidate)) return winCandidate; + const posixCandidate = join(dirname(process.execPath), '..', 'lib', 'node_modules', 'npm', 'bin', 'npm-cli.js'); + if (existsSync(posixCandidate)) return posixCandidate; + return null; +} + +/** + * Runs the sequence of hermetic test workspaces through zero-skip enforcement. + * + * @param {Object} [options={}] - Execution options. + * @param {string[]} [options.workspaces] - Workspaces to test (defaults to HERMETIC_WORKSPACES). + * @param {string} [options.npmCli] - Path to npm-cli.js. + * @param {boolean} [options.silent=false] - If true, suppresses stdout/stderr output. + * @param {function(string, string[], object): Promise} [options.runner=runWithZeroSkip] - Runner function. + * @param {function(string): { cmd: string, args: string[] }} [options.commandBuilder] - Custom command builder for testing. + * @returns {Promise} Resolves with 0 on complete zero-skip success, or non-zero on first failure. + */ +export async function runHermeticTests(options = {}) { + const workspaces = options.workspaces ?? HERMETIC_WORKSPACES; + const runner = options.runner ?? runWithZeroSkip; + const silent = Boolean(options.silent); + + let npmCli = options.npmCli; + if (!npmCli && !options.commandBuilder) { + npmCli = resolveNpmCli(); + if (!npmCli) { + if (!silent) { + process.stderr.write( + '[run-hermetic-tests] ERROR: npm_execpath is unavailable; please run via "npm test" or ensure npm is installed.\n' + ); + } + return 1; + } + } + + for (let i = 0; i < workspaces.length; i++) { + const ws = workspaces[i]; + if (!silent) { + process.stdout.write( + `\n\x1b[36m>>> [HERMETIC TEST ORCHESTRATOR] [${i + 1}/${workspaces.length}] Running workspace: ${ws}...\x1b[0m\n` + ); + } + + let cmd; + let args; + if (options.commandBuilder) { + const built = options.commandBuilder(ws); + cmd = built.cmd; + args = built.args; + } else { + cmd = process.execPath; + args = [npmCli, 'run', 'test', '--workspace', ws]; + } + + const code = await runner(cmd, args, { silent }); + if (code !== 0) { + if (!silent) { + process.stderr.write( + `\n\x1b[31m[HERMETIC TEST ORCHESTRATOR] FAILED on workspace "${ws}" with exit code ${code}.\x1b[0m\n` + ); + } + return code; + } + } + + if (!silent) { + process.stdout.write( + `\n\x1b[32m[HERMETIC TEST ORCHESTRATOR] SUCCESS: All ${workspaces.length} hermetic workspaces executed with zero skips.\x1b[0m\n` + ); + } + return 0; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + const cliWorkspaces = process.argv.slice(2).filter((arg) => !arg.startsWith('-')); + const workspaces = cliWorkspaces.length > 0 ? cliWorkspaces : undefined; + const code = await runHermeticTests({ workspaces }); + process.exit(code); +} diff --git a/scripts/run-live-provider-tests.mjs b/scripts/run-live-provider-tests.mjs new file mode 100644 index 00000000..c86731db --- /dev/null +++ b/scripts/run-live-provider-tests.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node +import { resolve } from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; +import { runWithZeroSkip } from './run-zero-skip.mjs'; + +const PROVIDERS = Object.freeze({ + openai: 'OPENAI_API_KEY', + anthropic: 'ANTHROPIC_API_KEY', +}); + +/** Selects only provisioned providers without ever reading or logging credential values. */ +export function selectLiveProviders(mode, env = process.env) { + if (mode !== 'all' && !(mode in PROVIDERS)) { + throw new Error(`Unknown live provider '${mode}'. Expected all, openai, or anthropic.`); + } + const candidates = mode === 'all' ? Object.keys(PROVIDERS) : [mode]; + const selected = candidates.filter((provider) => Boolean(env[PROVIDERS[provider]])); + if (selected.length === 0) { + const requirement = mode === 'all' + ? 'at least one of OPENAI_API_KEY or ANTHROPIC_API_KEY' + : PROVIDERS[mode]; + throw new Error(`Live provider tests require ${requirement}.`); + } + return selected; +} + +export async function runLiveProviderTests(mode, command, args, options = {}) { + const selected = selectLiveProviders(mode, options.env ?? process.env); + for (const provider of selected) { + const code = await runWithZeroSkip(command, args, { + ...options, + env: { + ...(options.env ?? process.env), + GAMBIT_LIVE_PROVIDER: provider, + }, + }); + if (code !== 0) return code; + } + return 0; +} + +const isCli = process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1]); +if (isCli) { + const [mode, separator, command, ...args] = process.argv.slice(2); + if (!mode || separator !== '--' || !command) { + console.error('Usage: node scripts/run-live-provider-tests.mjs -- [args...]'); + process.exit(1); + } + runLiveProviderTests(mode, command, args).then( + (code) => process.exit(code), + (error) => { + console.error(`[live-provider] ${error.message}`); + process.exit(1); + } + ); +} diff --git a/scripts/run-zero-skip.mjs b/scripts/run-zero-skip.mjs new file mode 100644 index 00000000..afec5818 --- /dev/null +++ b/scripts/run-zero-skip.mjs @@ -0,0 +1,193 @@ +#!/usr/bin/env node +/** + * Wraps a test command and fails if the test runner reports any skipped tests. + * + * Usage: + * node scripts/run-zero-skip.mjs [args...] + * node scripts/run-zero-skip.mjs -- npm test + */ +import { spawn } from 'node:child_process'; +import process from 'node:process'; +import { + createStreamingTestParser, + createStreamLineProcessor, +} from './lib/test-output-parser.mjs'; + +/** + * Spawns a test command, streams its output, and strictly enforces that the test + * runner reported at least one executed test and zero skipped tests. + * Maintains strictly O(1) bounded memory state by streaming lines directly to + * createStreamingTestParser, preventing transcript accumulation in memory. + * Uses independent StringDecoder instances and line buffers for stdout and stderr + * to prevent multibyte corruption and cross-stream line interleaving. + * + * @param {string} cmd - Command or binary to execute. + * @param {string[]} [args=[]] - Arguments to pass to the command. + * @param {import('node:child_process').SpawnOptions & { silent?: boolean }} [options={}] - Options for spawn and output suppression. + * @returns {Promise} Resolves with process exit code (0 on valid zero-skip pass, non-zero on failure). + */ +export function runWithZeroSkip(cmd, args = [], options = {}) { + return new Promise((resolve) => { + const child = spawn(cmd, args, { + stdio: ['inherit', 'pipe', 'pipe'], + shell: false, + ...options, + }); + + const parser = createStreamingTestParser(); + const processor = createStreamLineProcessor(parser); + const forwardedSignals = ['SIGINT', 'SIGTERM', 'SIGHUP']; + const signalHandlers = new Map(); + + for (const signal of forwardedSignals) { + const handler = () => { + if (!child.killed) child.kill(signal); + }; + signalHandlers.set(signal, handler); + process.on(signal, handler); + } + + const cleanupSignalHandlers = () => { + for (const [signal, handler] of signalHandlers) { + process.off(signal, handler); + } + }; + + child.stdout?.on('data', (chunk) => { + if (!options.silent) { + process.stdout.write(chunk); + } + processor.pushStdoutChunk(chunk); + }); + + child.stderr?.on('data', (chunk) => { + if (!options.silent) { + process.stderr.write(chunk); + } + processor.pushStderrChunk(chunk); + }); + + child.on('error', (err) => { + cleanupSignalHandlers(); + console.error(`[run-zero-skip] Failed to start process: ${err.message}`); + resolve(1); + }); + + child.on('close', (code, signal) => { + cleanupSignalHandlers(); + if (signal) { + if (!options.silent) { + process.stderr.write(`\n[run-zero-skip] Process terminated by signal: ${signal}\n`); + } + resolve(1); + return; + } + + if (code === null || code !== 0) { + resolve(code ?? 1); + return; + } + + const results = processor.getResults(); + + if (!results.accountingValid) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: Reporter accounting is incomplete or contradictory: ${results.accountingError}.\x1b[0m\n` + ); + } + resolve(1); + return; + } + + // Guard against missing or empty test runs (e.g. invalid glob, non-test command, or 0 tests executed). + // Only accept genuine line-anchored reporter summary lines (# tests N, ℹ tests N) or TAP plan headers (1..N). + // In multi-summary outputs, aggregate test counts and fail if any suite reports 0 executed tests. + if (results.totalTests === null || results.totalTests === 0) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: No executed tests detected in output (total=${results.totalTests}).\x1b[0m\n` + ); + } + resolve(1); + return; + } + + // Check for test runner skip indicators across TAP summaries and individual directives. + if (results.skippedCount > 0) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: Test suite finished with ${results.skippedCount} skipped test(s). The zero-skip policy requires skipped === 0.\x1b[0m\n` + ); + } + resolve(1); + return; + } + + // Check for TODO tests across TAP summaries and individual directives. + if (results.todoCount > 0) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: Test suite finished with ${results.todoCount} TODO test(s). The zero-skip policy requires todo === 0.\x1b[0m\n` + ); + } + resolve(1); + return; + } + + // Check for cancelled tests across TAP and spec format summaries. + if (results.cancelledCount > 0) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: Test suite finished with ${results.cancelledCount} cancelled test(s). The zero-skip policy requires cancelled === 0.\x1b[0m\n` + ); + } + resolve(1); + return; + } + + // Check for fail/failed summaries and raw TAP "not ok" test points. + if (results.failCount > 0) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: Test suite finished with ${results.failCount} failed test(s). The zero-skip policy requires fail === 0.\x1b[0m\n` + ); + } + resolve(1); + return; + } + + if (results.passCount === null || results.passCount !== results.totalTests) { + if (!options.silent) { + process.stderr.write( + `\n\x1b[31m[ZERO-SKIP ENFORCER] FAILED: Passing-test accounting does not equal the executed total (pass=${results.passCount}, total=${results.totalTests}).\x1b[0m\n` + ); + } + resolve(1); + return; + } + + resolve(0); + }); + }); +} + +// If invoked as CLI +const isCli = process.argv[1] && ( + process.argv[1].endsWith('run-zero-skip.mjs') || + import.meta.url.endsWith(process.argv[1].replace(/\\/g, '/')) +); + +if (isCli) { + const rawArgs = process.argv.slice(2); + const args = rawArgs[0] === '--' ? rawArgs.slice(1) : rawArgs; + if (args.length === 0) { + console.error('Usage: node scripts/run-zero-skip.mjs [--] [args...]'); + process.exit(1); + } + + const [cmd, ...cmdArgs] = args; + runWithZeroSkip(cmd, cmdArgs).then((code) => { + process.exit(code); + }); +} diff --git a/scripts/test-counts.mjs b/scripts/test-counts.mjs index 6e234819..6de042d5 100644 --- a/scripts/test-counts.mjs +++ b/scripts/test-counts.mjs @@ -1,8 +1,10 @@ #!/usr/bin/env node -/** Run every suite through npm and report Node test-runner totals portably. */ +/** Run test suites through npm and report Node test-runner totals portably with zero-skip auditing. */ import { spawnSync } from 'node:child_process'; import { dirname, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; +import { parseCompleteTestOutput } from './lib/test-output-parser.mjs'; +import { getHermeticWorkspaces, PARTITIONED_PACKAGES } from './lib/workspace-topology.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); const npmCli = process.env.npm_execpath; @@ -10,68 +12,221 @@ if (!npmCli) { console.error('npm_execpath is unavailable; run this script through "npm run test:counts"'); process.exit(1); } -const suites = [ - ['core', ['test', '--workspace', '@chess-platform/core']], - ['search', ['test', '--workspace', '@chess-platform/search']], - ['social', ['test', '--workspace', '@chess-platform/social']], - ['messaging', ['test', '--workspace', '@chess-platform/messaging']], - ['community', ['test', '--workspace', '@chess-platform/community']], - ['achievements', ['test', '--workspace', '@chess-platform/achievements']], - ['studies', ['test', '--workspace', '@chess-platform/studies']], - ['learning', ['test', '--workspace', '@chess-platform/learning']], - ['anti-cheat', ['test', '--workspace', '@chess-platform/anti-cheat']], - ['game', ['test', '--workspace', '@chess-platform/game']], - ['tournament', ['test', '--workspace', '@chess-platform/tournament']], - ['realtime-gateway', ['test', '--workspace', '@chess-platform/realtime-gateway']], - ['persistence', ['test', '--workspace', '@chess-platform/persistence']], - ['api', ['test', '--workspace', '@chess-platform/api']], - ['engine', ['test', '--workspace', '@chess-platform/engine']], - ['web', ['test', '--workspace', '@chess-platform/web']], - ['e2e-harness', ['test', '--workspace', '@chess-platform/e2e-harness']], - ['ai-orchestrator', ['test', '--workspace', '@chess-platform/ai-orchestrator']], - ['ai-features', ['test', '--workspace', '@chess-platform/ai-features']], - ['gateway-service', ['test', '--prefix', 'services/gateway']], + +const HERMETIC_SUITES = [ + ...getHermeticWorkspaces().map((pkg) => { + const base = pkg.replace('@chess-platform/', ''); + const label = PARTITIONED_PACKAGES.includes(pkg) ? `${base} (hermetic unit)` : base; + return [label, ['test', '--workspace', pkg]]; + }), + ['scripts (hermetic unit)', ['run', 'test:scripts']], + ['load-harness (hermetic unit)', ['run', 'test:load-harness']], +]; + +const SERVICE_SUITES = [ + { + name: 'gateway-service (redis integration)', + args: ['test', '--prefix', 'services/gateway'], + envReq: 'REDIS_URL', + isAvailable: Boolean(process.env.REDIS_URL), + }, + { + name: 'gateway-service (trusted edge through Nginx)', + args: ['run', 'test:trusted-edge', '--prefix', 'services/gateway'], + envReq: 'REQUIRE_DOCKER=1', + isAvailable: process.env.REQUIRE_DOCKER === '1', + env: { REQUIRE_DOCKER: '1' }, + }, + { + name: 'gateway-service (web delivery through Nginx)', + args: ['run', 'test:web-delivery', '--prefix', 'services/gateway'], + envReq: 'REQUIRE_DOCKER=1', + isAvailable: process.env.REQUIRE_DOCKER === '1', + env: { REQUIRE_DOCKER: '1' }, + }, + { + name: 'web (Playwright acceptance)', + args: ['run', 'e2e', '--workspace', '@chess-platform/web'], + envReq: 'GAMBIT_E2E_BACKEND=1 and Playwright Chromium installed', + isAvailable: process.env.GAMBIT_E2E_BACKEND === '1', + env: { GAMBIT_E2E_BACKEND: '1' }, + }, + { + name: 'persistence (postgres integration)', + args: ['run', 'test:integration:postgres', '--workspace', '@chess-platform/persistence'], + envReq: 'DATABASE_URL', + isAvailable: Boolean(process.env.DATABASE_URL), + }, + { + name: 'api (postgres integration)', + args: ['run', 'test:integration:postgres', '--workspace', '@chess-platform/api'], + envReq: 'DATABASE_URL', + isAvailable: Boolean(process.env.DATABASE_URL), + }, + { + name: 'scripts (postgres integration)', + args: ['run', 'test:scripts:integration'], + envReq: 'DATABASE_URL', + isAvailable: Boolean(process.env.DATABASE_URL), + }, + { + name: 'api (engine smoke)', + args: ['run', 'test:analysis-smoke', '--workspace', '@chess-platform/api'], + envReq: 'STOCKFISH_PATH & DATABASE_URL', + isAvailable: Boolean(process.env.STOCKFISH_PATH && process.env.DATABASE_URL), + }, + { + name: 'ai-orchestrator (OpenAI live contract)', + args: ['run', 'test:live-provider:openai', '--workspace', '@chess-platform/ai-orchestrator'], + envReq: 'OPENAI_API_KEY', + isAvailable: Boolean(process.env.OPENAI_API_KEY), + }, + { + name: 'ai-orchestrator (Anthropic live contract)', + args: ['run', 'test:live-provider:anthropic', '--workspace', '@chess-platform/ai-orchestrator'], + envReq: 'ANTHROPIC_API_KEY', + isAvailable: Boolean(process.env.ANTHROPIC_API_KEY), + }, + { + name: 'ai-features (OpenAI live contract)', + args: ['run', 'test:live-provider:openai', '--workspace', '@chess-platform/ai-features'], + envReq: 'OPENAI_API_KEY', + isAvailable: Boolean(process.env.OPENAI_API_KEY), + }, + { + name: 'ai-features (Anthropic live contract)', + args: ['run', 'test:live-provider:anthropic', '--workspace', '@chess-platform/ai-features'], + envReq: 'ANTHROPIC_API_KEY', + isAvailable: Boolean(process.env.ANTHROPIC_API_KEY), + }, + { + name: 'api (posix unit)', + args: ['run', 'test:posix', '--workspace', '@chess-platform/api'], + envReq: 'POSIX platform required', + isAvailable: process.platform !== 'win32', + }, + { + name: 'load-harness (posix)', + args: ['run', 'test:load-harness:posix'], + envReq: 'POSIX platform required', + isAvailable: process.platform !== 'win32', + }, ]; -let total = 0; -let skippedTotal = 0; +let totalTests = 0; +let totalPassed = 0; +let totalFailed = 0; +let totalSkipped = 0; +let totalTodo = 0; +let totalCancelled = 0; let hadError = false; -for (const [name, args] of suites) { +console.log('\n=== Hermetic Unit Test Suites ==='); +for (const [name, args] of HERMETIC_SUITES) { const result = spawnSync(process.execPath, [npmCli, ...args], { cwd: root, encoding: 'utf8', windowsHide: true, env: process.env, + maxBuffer: 64 * 1024 * 1024, }); const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`; - const tests = metric(output, 'tests'); - const skipped = metric(output, 'skipped') ?? 0; - const failed = metric(output, 'fail') ?? 0; + const parsed = parseCompleteTestOutput(output); + const { totalTests: tests, passCount: passed, failCount: failed, skippedCount: skipped, + todoCount: todo, cancelledCount: cancelled, accountingValid, accountingError } = parsed; - if (result.status !== 0 || tests === null || failed > 0) { - console.error(`${name}: ERROR`); + if ( + result.status !== 0 || + result.error || + tests === null || + tests === 0 || + !accountingValid || + passed === null || + passed !== tests || + failed > 0 || + skipped > 0 || + todo > 0 || + cancelled > 0 + ) { + if (result.error?.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + console.error(`${name}: ERROR (output exceeded maxBuffer of 64MB)`); + } else { + console.error( + `${name}: ERROR (status=${result.status}, tests=${tests}, passed=${passed}, failed=${failed}, skipped=${skipped}, todo=${todo}, cancelled=${cancelled}, accounting=${accountingError ?? 'valid'})` + ); + } if (result.error) console.error(result.error.message); if (output.trim()) console.error(output.trim()); hadError = true; continue; } - console.log(`${name}: ${tests} tests (${skipped} skipped)`); - total += tests; - skippedTotal += skipped; + console.log( + `${name}: tests ${tests}, passed ${passed}, failed ${failed}, skipped ${skipped}, todo ${todo}, cancelled ${cancelled}` + ); + totalTests += tests; + totalPassed += passed; + totalFailed += failed; + totalSkipped += skipped; + totalTodo += todo; + totalCancelled += cancelled; } -console.log(`Total: ${total} tests (${skippedTotal} skipped)`); -if (hadError) process.exitCode = 1; +console.log('\n=== Environment & Integration Test Suites ==='); +for (const suite of SERVICE_SUITES) { + if (!suite.isAvailable) { + console.log(`${suite.name}: NOT EXECUTED (${suite.envReq} not provided)`); + continue; + } + + const result = spawnSync(process.execPath, [npmCli, ...suite.args], { + cwd: root, + encoding: 'utf8', + windowsHide: true, + env: { ...process.env, ...(suite.env ?? {}) }, + maxBuffer: 64 * 1024 * 1024, + }); + const output = `${result.stdout ?? ''}\n${result.stderr ?? ''}`; + const parsed = parseCompleteTestOutput(output); + const { totalTests: tests, passCount: passed, failCount: failed, skippedCount: skipped, + todoCount: todo, cancelledCount: cancelled, accountingValid, accountingError } = parsed; -function metric(output, name) { - const patterns = [ - new RegExp(`^# ${name}\\s+(\\d+)`, 'm'), - new RegExp(`^ℹ ${name}\\s+(\\d+)`, 'm'), - ]; - for (const pattern of patterns) { - const match = pattern.exec(output); - if (match) return Number(match[1]); + if ( + result.status !== 0 || + result.error || + tests === null || + tests === 0 || + !accountingValid || + passed === null || + passed !== tests || + failed > 0 || + skipped > 0 || + todo > 0 || + cancelled > 0 + ) { + if (result.error?.code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + console.error(`${suite.name}: ERROR (output exceeded maxBuffer of 64MB)`); + } else { + console.error( + `${suite.name}: ERROR (status=${result.status}, tests=${tests}, passed=${passed}, failed=${failed}, skipped=${skipped}, todo=${todo}, cancelled=${cancelled}, accounting=${accountingError ?? 'valid'})` + ); + } + if (result.error) console.error(result.error.message); + if (output.trim()) console.error(output.trim()); + hadError = true; + continue; } - return null; + console.log( + `${suite.name}: tests ${tests}, passed ${passed}, failed ${failed}, skipped ${skipped}, todo ${todo}, cancelled ${cancelled}` + ); + totalTests += tests; + totalPassed += passed; + totalFailed += failed; + totalSkipped += skipped; + totalTodo += todo; + totalCancelled += cancelled; } + +console.log( + `\nGrand Total Executed: ${totalTests} tests (${totalPassed} passed, ${totalFailed} failed, ${totalSkipped} skipped, ${totalTodo} todo, ${totalCancelled} cancelled)` +); +if (hadError) process.exitCode = 1; diff --git a/scripts/test/backup-restore-drill.integration.test.mjs b/scripts/test/backup-restore-drill.integration.test.mjs new file mode 100644 index 00000000..48f73cc5 --- /dev/null +++ b/scripts/test/backup-restore-drill.integration.test.mjs @@ -0,0 +1,28 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { withTestDatabase } from '@chess-platform/persistence/test-support'; +import { migrate } from '@chess-platform/persistence/pg'; +import { runBackupRestoreDrill } from '../db-backup-restore-drill.mjs'; + +const migrationsDir = resolve(fileURLToPath(import.meta.url), '../../../packages/persistence/migrations'); + +test( + 'integration: full backup, isolated restore, and verification drill against live Postgres', + async () => { + assert.ok(process.env.DATABASE_URL, 'DATABASE_URL is required for live Postgres integration drill'); + await withTestDatabase(async ({ pool, connectionString }) => { + pool.on('error', () => {}); + await migrate(pool, migrationsDir); + await pool.end(); + const report = await runBackupRestoreDrill({ + sourceUrl: connectionString, + keepTarget: false, + keepBackup: false, + }); + assert.equal(report.success, true); + assert.ok(report.checks.length > 0); + }); + }, +); diff --git a/scripts/test/backup-restore-drill.test.mjs b/scripts/test/backup-restore-drill.test.mjs index 11ffa89d..869dc787 100644 --- a/scripts/test/backup-restore-drill.test.mjs +++ b/scripts/test/backup-restore-drill.test.mjs @@ -638,18 +638,3 @@ test('verification engine: checks REQUIRED_EXTENSIONS even when not present in s }, ); }); - -test( - 'integration: full backup, isolated restore, and verification drill against live Postgres', - { skip: process.env.DATABASE_URL ? false : 'DATABASE_URL not set' }, - async () => { - const sourceUrl = process.env.DATABASE_URL; - const report = await runBackupRestoreDrill({ - sourceUrl, - keepTarget: false, - keepBackup: false, - }); - assert.equal(report.success, true); - assert.ok(report.checks.length > 0); - }, -); diff --git a/scripts/test/check-test-topology.test.mjs b/scripts/test/check-test-topology.test.mjs new file mode 100644 index 00000000..98fa2198 --- /dev/null +++ b/scripts/test/check-test-topology.test.mjs @@ -0,0 +1,624 @@ +import { test } from 'node:test'; +import * as assert from 'node:assert/strict'; +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { + findTestFiles, + classifyTestFile, + verifyTestTopology, + isAllowedPlacement, + extractRunnerPatterns, + extractPlaywrightPatterns, + getPlaywrightDiscoveredFiles, + matchRunnerPattern, + matchRunnerPatternFallback, + isTestFileReachableByRunner, + SUITE_DEFINITIONS, +} from '../check-test-topology.mjs'; +import { + discoverWorkspacePackages, + getHermeticWorkspaces, +} from '../lib/workspace-topology.mjs'; + + +test('topology: all test files in the repository are classified into explicit suites', () => { + const result = verifyTestTopology(); + assert.equal(result.misplaced.length, 0, `Found misplaced test files: ${result.misplaced.join(', ')}`); + assert.equal(result.unclassified.length, 0, `Found unclassified test files: ${result.unclassified.join(', ')}`); + assert.equal(result.unreachable.length, 0, `Found unreachable test files: ${result.unreachable.map((u) => `${u.file} (${u.suite})`).join(', ')}`); + assert.ok(result.totalFiles > 300, `Expected over 300 test files, got ${result.totalFiles}`); +}); + +test('topology: isAllowedPlacement rejects misplaced test files in unauthorized locations', () => { + assert.equal(isAllowedPlacement('packages/chess-core/src/fen.test.ts'), false); + assert.equal(isAllowedPlacement('test/root.test.ts'), false); + assert.equal(isAllowedPlacement('root.test.js'), false); + assert.equal(isAllowedPlacement('services/gateway/src/server.test.ts'), false); + + assert.equal(isAllowedPlacement('packages/chess-core/test/fen.test.ts'), true); + assert.equal(isAllowedPlacement('packages/web/e2e/game.spec.ts'), true); + assert.equal(isAllowedPlacement('services/gateway/test/engine-bot.test.ts'), true); + assert.equal(isAllowedPlacement('scripts/test/zero-skip-enforcement.test.mjs'), true); + assert.equal(isAllowedPlacement('scripts/nginx-trusted-edge-acceptance.mjs'), true); + assert.equal(isAllowedPlacement('scripts/nginx-web-delivery-acceptance.mjs'), true); + assert.equal(isAllowedPlacement('deploy/load/test/run-evidence.test.mjs'), true); +}); + +test('topology: classifyTestFile correctly maps each suite pattern', () => { + assert.equal(classifyTestFile('packages/web/e2e/game.spec.ts')?.name, 'acceptance-playwright'); + assert.equal(classifyTestFile('services/gateway/test/redis-ownership.integration.test.ts')?.name, 'gateway-redis-integration'); + assert.equal(classifyTestFile('services/gateway/test/engine-bot.test.ts')?.name, 'gateway-unit'); + assert.equal(classifyTestFile('scripts/nginx-trusted-edge-acceptance.mjs')?.name, 'gateway-trusted-edge'); + assert.equal(classifyTestFile('scripts/nginx-web-delivery-acceptance.mjs')?.name, 'gateway-web-delivery'); + assert.equal(classifyTestFile('packages/persistence/test/pg.integration.test.ts')?.name, 'persistence-postgres-integration'); + assert.equal(classifyTestFile('packages/persistence/test/event-store.test.ts')?.name, 'persistence-unit'); + assert.equal(classifyTestFile('packages/api/test/pg-security.integration.test.ts')?.name, 'api-postgres-integration'); + assert.equal(classifyTestFile('packages/api/test/analysis-stockfish-smoke.test.ts')?.name, 'api-engine-smoke'); + assert.equal(classifyTestFile('packages/api/test/diagnostics/signature-b-preload-abort.diag.ts')?.name, 'api-diagnostics'); + assert.equal(classifyTestFile('packages/api/test/diagnostics/signature-b-correlate.posix.test.ts')?.name, 'api-posix-unit'); + assert.equal(classifyTestFile('packages/api/test/auth.test.ts')?.name, 'api-unit'); + assert.equal(classifyTestFile('packages/ai-orchestrator/test/adapters-live.integration.test.ts')?.name, 'ai-orchestrator-live-provider'); + assert.equal(classifyTestFile('packages/ai-orchestrator/test/orchestrator.test.ts')?.name, 'ai-orchestrator-unit'); + assert.equal(classifyTestFile('packages/ai-features/test/coach-integration.test.ts')?.name, 'ai-features-live-provider'); + assert.equal(classifyTestFile('packages/ai-features/test/coach.test.ts')?.name, 'ai-features-unit'); + assert.equal(classifyTestFile('scripts/test/backup-restore-drill.integration.test.mjs')?.name, 'scripts-postgres-integration'); + assert.equal(classifyTestFile('scripts/test/zero-skip-enforcement.test.mjs')?.name, 'scripts-unit'); + assert.equal(classifyTestFile('deploy/load/test/run-evidence.posix.test.mjs')?.name, 'load-harness-posix'); + assert.equal(classifyTestFile('deploy/load/test/run-evidence.test.mjs')?.name, 'load-harness-unit'); + assert.equal(classifyTestFile('packages/chess-core/test/fen.test.ts')?.name, 'domain-hermetic-unit'); +}); + +test('topology: extractRunnerPatterns mechanically extracts globs and files from package runner scripts', () => { + assert.deepEqual( + extractRunnerPatterns('node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 "dist-test/test/**/*.posix.test.js"'), + ['dist-test/test/**/*.posix.test.js'] + ); + assert.deepEqual( + extractRunnerPatterns('tsc -p tsconfig.test.json && node ../../scripts/run-zero-skip.mjs -- node --test dist-test/test/a.test.js dist-test/test/b.test.js'), + ['dist-test/test/a.test.js', 'dist-test/test/b.test.js'] + ); + // Mechanically extracts from packages/web/playwright.config.ts testDir ('./e2e') + assert.deepEqual( + extractRunnerPatterns('playwright test', { manifestDir: 'packages/web' }), + ['e2e/**/*.spec.ts'] + ); + assert.deepEqual( + extractPlaywrightPatterns('packages/web'), + ['e2e/**/*.spec.ts'] + ); + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': "testDir: './smoke-e2e', testMatch: '**/*.acceptance.ts'", + }, + }), + ['smoke-e2e/**/*.acceptance.ts'] + ); + assert.deepEqual(extractRunnerPatterns('echo "not a test runner"'), []); +}); + + +test('topology: runner reachability mechanically validates package manifest configuration', () => { + const smokeSuite = SUITE_DEFINITIONS.find((s) => s.name === 'api-engine-smoke'); + assert.ok(smokeSuite?.isReachable); + assert.equal(smokeSuite.isReachable('packages/api/test/analysis-stockfish-smoke.test.ts'), true); + assert.equal(smokeSuite.isReachable('packages/api/test/analysis-unknown-smoke.test.ts'), false); + + const persistenceUnitSuite = SUITE_DEFINITIONS.find((s) => s.name === 'persistence-unit'); + assert.ok(persistenceUnitSuite?.isReachable); + assert.equal(persistenceUnitSuite.isReachable('packages/persistence/test/event-store.test.ts'), true); + assert.equal(persistenceUnitSuite.isReachable('packages/persistence/test/pg.integration.test.ts'), false); + + const apiUnitSuite = SUITE_DEFINITIONS.find((s) => s.name === 'api-unit'); + assert.ok(apiUnitSuite?.isReachable); + assert.equal(apiUnitSuite.isReachable('packages/api/test/auth.test.ts'), true); + assert.equal(apiUnitSuite.isReachable('packages/api/test/analysis-stockfish-smoke.test.ts'), false); + assert.equal(apiUnitSuite.isReachable('packages/api/test/diagnostics/signature-b-correlate.posix.test.ts'), false); + assert.equal(apiUnitSuite.isReachable('packages/api/test/pg-security.integration.test.ts'), false); +}); + +test('topology: falsification regression proves validation fails when runner glob narrows', () => { + // Falsification Case 1: Narrowing api test:posix in manifest causes nested posix test to become unreachable + const falsifiedPosix = verifyTestTopology(undefined, { + manifestOverrides: { + 'packages/api/package.json': { + scripts: { + 'test:posix': 'npm run build:test && node ../../scripts/run-zero-skip.mjs -- node --test --test-concurrency=1 "dist-test/test/*.posix.test.js"', + }, + }, + }, + }); + assert.ok(falsifiedPosix.unreachable.length > 0, 'Topology check must fail when runner glob is non-recursive'); + const posixFailure = falsifiedPosix.unreachable.find((u) => u.file === 'packages/api/test/diagnostics/signature-b-correlate.posix.test.ts'); + assert.ok(posixFailure, 'signature-b-correlate.posix.test.ts must be flagged unreachable when glob does not recurse'); + assert.equal(posixFailure.suite, 'api-posix-unit'); + + // Falsification Case 2: Narrowing test:scripts in root manifest causes unlisted scripts test to become unreachable + const falsifiedScripts = verifyTestTopology(undefined, { + manifestOverrides: { + 'package.json': { + scripts: { + 'test:scripts': 'node scripts/run-zero-skip.mjs -- node --test "scripts/test/zero-skip-enforcement.test.mjs"', + }, + }, + }, + }); + assert.ok(falsifiedScripts.unreachable.length > 0, 'Topology check must fail when root script runner is narrowed'); + const scriptsFailure = falsifiedScripts.unreachable.find((u) => u.file === 'scripts/test/check-test-topology.test.mjs'); + assert.ok(scriptsFailure, 'check-test-topology.test.mjs must be flagged unreachable when test:scripts is narrowed'); + assert.equal(scriptsFailure.suite, 'scripts-unit'); + + // Falsification Case 3: Narrowing persistence test:unit to a non-existent pattern causes all persistence tests to be unreachable + const falsifiedPersistence = verifyTestTopology(undefined, { + manifestOverrides: { + 'packages/persistence/package.json': { + scripts: { + 'test:unit': 'node ../../scripts/run-zero-skip.mjs -- node --test "dist-test/test/none.test.js"', + }, + }, + }, + }); + assert.ok(falsifiedPersistence.unreachable.length >= 7, 'All persistence unit tests must be flagged unreachable'); + assert.ok(falsifiedPersistence.unreachable.every((u) => u.suite === 'persistence-unit')); +}); + +test('topology: classifyTestFile returns null for unknown files', () => { + assert.equal(classifyTestFile('random/path/unknown.test.ts'), null); +}); + +test('topology: falsification regression proves validation fails when Playwright testDir drifts', () => { + // Falsification Case 4: Changing testDir in Playwright config causes web e2e tests to become unreachable + const falsifiedPlaywright = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + import type { PlaywrightTestConfig } from '@playwright/test'; + const config: PlaywrightTestConfig = { + testDir: './drifted-e2e', + }; + export default config; + `, + }, + }); + assert.ok(falsifiedPlaywright.unreachable.length >= 25, 'Topology check must fail when Playwright testDir drifts'); + assert.ok(falsifiedPlaywright.unreachable.every((u) => u.suite === 'acceptance-playwright')); + const webFailure = falsifiedPlaywright.unreachable.find((u) => u.file === 'packages/web/e2e/game-actions.spec.ts'); + assert.ok(webFailure, 'packages/web/e2e/game-actions.spec.ts must be flagged unreachable when Playwright testDir is changed to ./drifted-e2e'); + assert.equal(webFailure.suite, 'acceptance-playwright'); +}); + +test('workspace topology: filters hermetic workspaces by packages/ filesystem location and derives relDir dynamically', () => { + const tmpDir = mkdtempSync(join(tmpdir(), 'synth-workspace-')); + try { + writeFileSync( + join(tmpDir, 'package.json'), + JSON.stringify({ + name: 'synthetic-monorepo', + workspaces: ['packages/*', 'services/*', 'tools/cli'], + }) + ); + mkdirSync(join(tmpDir, 'packages', 'core'), { recursive: true }); + writeFileSync( + join(tmpDir, 'packages', 'core', 'package.json'), + JSON.stringify({ name: '@chess-platform/core' }) + ); + mkdirSync(join(tmpDir, 'services', 'gateway'), { recursive: true }); + writeFileSync( + join(tmpDir, 'services', 'gateway', 'package.json'), + JSON.stringify({ name: '@chess-platform/gateway' }) + ); + mkdirSync(join(tmpDir, 'tools', 'cli'), { recursive: true }); + writeFileSync( + join(tmpDir, 'tools', 'cli', 'package.json'), + JSON.stringify({ name: 'custom-cli' }) + ); + + const discovered = discoverWorkspacePackages(tmpDir); + assert.equal(discovered.length, 3); + const corePkg = discovered.find((p) => p.name === '@chess-platform/core'); + const gatewayPkg = discovered.find((p) => p.name === '@chess-platform/gateway'); + const cliPkg = discovered.find((p) => p.name === 'custom-cli'); + + // relDir must be derived from actual relative path, not hardcoded packages/* + assert.equal(corePkg?.relDir, 'packages/core'); + assert.equal(gatewayPkg?.relDir, 'services/gateway'); + assert.equal(cliPkg?.relDir, 'tools/cli'); + + // getHermeticWorkspaces must ONLY include packages physically located under packages/ + const hermetic = getHermeticWorkspaces(tmpDir); + assert.deepEqual([...hermetic], ['@chess-platform/core']); + assert.ok(!hermetic.includes('@chess-platform/gateway'), 'services/* workspace must not be included in hermetic fan-out'); + assert.ok(!hermetic.includes('custom-cli'), 'tools/* workspace must not be included in hermetic fan-out'); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test('topology: matchRunnerPattern correctly matches glob patterns across directory levels', () => { + assert.equal(matchRunnerPattern('e2e/**/*.spec.ts', 'e2e/game-actions.spec.ts'), true); + assert.equal(matchRunnerPattern('e2e/**/*.spec.ts', 'e2e/nested/deep/game-actions.spec.ts'), true); + assert.equal(matchRunnerPattern('e2e/**/*.spec.ts', 'packages/web/e2e/game-actions.spec.ts'), false); + assert.equal(matchRunnerPattern('dist-test/test/**/*.posix.test.js', 'dist-test/test/diagnostics/signature-b-correlate.posix.test.js'), true); + assert.equal(matchRunnerPattern('dist-test/test/**/*.posix.test.js', 'dist-test/test/foo.posix.test.js'), true); +}); + +test('topology: regex fallback matches **/ as zero or more directory segments', () => { + const normPattern = 'e2e/**/*.spec.ts'; + const reStr = normPattern + .replace(/[.+^${}()|[\]\\]/g, '\\$&') + .replace(/\*\*\/|\*\*|\*/g, (token) => { + if (token === '**/') return '(?:[^/]+/)*'; + if (token === '**') return '.*'; + return '[^/]*'; + }); + const re = new RegExp(`^${reStr}$`); + assert.equal(re.test('e2e/game-actions.spec.ts'), true, 'zero directory segments under e2e must match'); + assert.equal(re.test('e2e/nested/game-actions.spec.ts'), true, 'one directory segment under e2e must match'); + assert.equal(re.test('e2e/nested/deep/game-actions.spec.ts'), true, 'multiple directory segments under e2e must match'); + assert.equal(re.test('other/game-actions.spec.ts'), false, 'different directory must not match'); +}); + +test('topology: portable glob fallback supports the repository negative extglob', () => { + const pattern = 'dist-test/test/**/!(*integration).test.js'; + assert.equal(matchRunnerPatternFallback(pattern, 'dist-test/test/unit.test.js'), true); + assert.equal(matchRunnerPatternFallback(pattern, 'dist-test/test/nested/unit.test.js'), true); + assert.equal(matchRunnerPatternFallback(pattern, 'dist-test/test/provider.integration.test.js'), false); +}); + +test('topology: deployment exclusions do not hide colliding package directories', () => { + const tmpDir = mkdtempSync(join(tmpdir(), 'topology-exclusions-')); + try { + const visible = [ + 'packages/example/helm/test/visible.test.ts', + 'packages/example/observability/test/visible.test.ts', + ]; + const excluded = [ + 'deploy/helm/test/chart.test.ts', + 'deploy/observability/test/dashboard.test.ts', + ]; + for (const relPath of [...visible, ...excluded]) { + const fullPath = join(tmpDir, ...relPath.split('/')); + mkdirSync(dirname(fullPath), { recursive: true }); + writeFileSync(fullPath, ''); + } + const found = findTestFiles(tmpDir); + assert.deepEqual(found, visible); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +}); + +test('topology: extractPlaywrightPatterns confidently resolves literal testDir and defaults testMatch when omitted', () => { + // Case 1: Real project config (testDir: './e2e', testMatch omitted) + assert.deepEqual( + extractPlaywrightPatterns('packages/web'), + ['e2e/**/*.spec.ts'] + ); + + // Case 2: testDir omitted, testMatch omitted -> real Playwright defaults used + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + timeout: 30000, + }; + `, + }, + }), + ['**/*.spec.ts'] + ); + + // Case 3: testDir omitted, testMatch provided as literal + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + testMatch: '**/*.acceptance.ts', + }; + `, + }, + }), + ['**/*.acceptance.ts'] + ); + + // Case 4: testDir provided, testMatch provided as array of string literals + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + testDir: './e2e', + testMatch: [ + '**/*.spec.ts', + '**/*.acceptance.ts', + ], + }; + `, + }, + }), + ['e2e/**/*.spec.ts', 'e2e/**/*.acceptance.ts'] + ); + + // Case 5: Comments containing testDir or testMatch are ignored + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + // testDir: unparseableVariable, + /* testMatch: unparseableMatch, */ + export default { + testDir: './e2e', + }; + `, + }, + }), + ['e2e/**/*.spec.ts'] + ); + + // Case 6: Preceding string literals containing testMatch or testDir are ignored + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + const note = "Note that testMatch: '**/*.legacy.ts' was deprecated"; + export default { + testDir: './e2e', + }; + `, + }, + }), + ['e2e/**/*.spec.ts'] + ); + + // Case 7: Nested project properties do not override direct config properties + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + projects: [ + { + name: 'legacy', + testDir: './nested-legacy', + testMatch: '**/*.legacy.ts', + }, + ], + testDir: './e2e', + }; + `, + }, + }), + ['e2e/**/*.spec.ts'] + ); + + // Case 8: Direct property value containing property keywords in string literals is safely skipped + assert.deepEqual( + extractPlaywrightPatterns('packages/web', { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + name: "e2e runner with testMatch: '**/*.decoy.ts'", + testDir: './e2e', + }; + `, + }, + }), + ['e2e/**/*.spec.ts'] + ); +}); + +test('topology: extractPlaywrightPatterns fails closed on non-literal static expressions', () => { + const overrideVar = { + 'packages/web/playwright.config.ts': ` + const dir = './other-e2e'; + export default { testDir: dir }; + `, + }; + assert.throws( + () => extractPlaywrightPatterns('packages/web', { playwrightConfigOverrides: overrideVar }), + /Cannot mechanically resolve Playwright 'testDir'.*property is present but non-literal or unparseable/ + ); + + const overrideExpr = { + 'packages/web/playwright.config.ts': ` + export default { + testDir: path.join(__dirname, 'e2e'), + }; + `, + }; + assert.throws( + () => extractPlaywrightPatterns('packages/web', { playwrightConfigOverrides: overrideExpr }), + /Cannot mechanically resolve Playwright 'testDir'/ + ); + + const overrideMatchVar = { + 'packages/web/playwright.config.ts': ` + const customMatch = '**/*.spec.ts'; + export default { + testDir: './e2e', + testMatch: customMatch, + }; + `, + }; + assert.throws( + () => extractPlaywrightPatterns('packages/web', { playwrightConfigOverrides: overrideMatchVar }), + /Cannot mechanically resolve Playwright 'testMatch'.*property is present but non-literal or unsupported/ + ); +}); + +test('topology: getPlaywrightDiscoveredFiles derives reachable files directly from Playwright CLI', () => { + const discovered = getPlaywrightDiscoveredFiles('packages/web'); + assert.equal(discovered.size, 26); + assert.ok(discovered.has('packages/web/e2e/game-actions.spec.ts')); + assert.ok(discovered.has('packages/web/e2e/app-loads.spec.ts')); +}); + +test('topology: backend-free Playwright discovers only the seven offline specs', () => { + const offline = getPlaywrightDiscoveredFiles('packages/web', { backend: false }); + const full = getPlaywrightDiscoveredFiles('packages/web'); + assert.equal(offline.size, 7); + assert.ok(offline.has('packages/web/e2e/app-loads.spec.ts')); + assert.ok(!offline.has('packages/web/e2e/game-actions.spec.ts')); + assert.equal(full.size, 26); +}); + +test('topology: falsification regression proves validation flags ignored test files unreachable', () => { + const falsified = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + testDir: './e2e', + testIgnore: '**/game-actions.spec.ts', + }; + `, + }, + }); + assert.ok(falsified.unreachable.length > 0, 'Test file excluded by testIgnore must be unreachable'); + const ignoredFailure = falsified.unreachable.find((u) => u.file === 'packages/web/e2e/game-actions.spec.ts'); + assert.ok(ignoredFailure, 'game-actions.spec.ts must be flagged unreachable'); + assert.equal(ignoredFailure.suite, 'acceptance-playwright'); +}); + +test('topology: falsification regression proves project-level testDir overrides are respected', () => { + const falsified = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + projects: [ + { + name: 'other', + testDir: './drifted-e2e', + }, + ], + }; + `, + }, + }); + assert.ok(falsified.unreachable.length >= 25, 'Files outside project testDir must be unreachable'); + assert.ok(falsified.unreachable.every((u) => u.suite === 'acceptance-playwright')); +}); + +test('topology: falsification regression proves project-level testIgnore excludes files', () => { + const falsified = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + testDir: './e2e', + projects: [ + { + name: 'chromium', + testIgnore: '**/game-actions.spec.ts', + }, + ], + }; + `, + }, + }); + const ignoredFailure = falsified.unreachable.find((u) => u.file === 'packages/web/e2e/game-actions.spec.ts'); + assert.ok(ignoredFailure, 'game-actions.spec.ts must be flagged unreachable by project testIgnore'); +}); + +test('topology: object spreads and dynamic variables in Playwright config are evaluated by Playwright discovery engine', () => { + // Case A: variable testDir pointing to drifted directory flags e2e files unreachable + const falsifiedVar = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + const dir = './other-e2e'; + export default { testDir: dir }; + `, + }, + }); + assert.ok(falsifiedVar.unreachable.length >= 25, 'Variable testDir pointing to ./other-e2e must flag e2e files unreachable'); + + // Case B: spread config pointing to ./e2e discovers all files + const spreadSuccess = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + const base = { testDir: './e2e' }; + export default { ...base }; + `, + }, + }); + assert.equal(spreadSuccess.unreachable.length, 0, 'Spread config pointing to ./e2e must reach all test files'); + + // Case C: spread config pointing to ./other-e2e flags e2e files unreachable + const spreadDrift = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + const base = { testDir: './other-e2e' }; + export default { ...base }; + `, + }, + }); + assert.ok(spreadDrift.unreachable.length >= 25, 'Spread config pointing to ./other-e2e must flag e2e files unreachable'); +}); + +test('topology: omitting testMatch uses real Playwright default pattern', () => { + const defaultMatch = verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + testDir: './e2e', + }; + `, + }, + }); + assert.equal(defaultMatch.unreachable.length, 0, 'Omitting testMatch must use Playwright default and discover all e2e spec files'); +}); + +test('topology: unresolvable or errored Playwright configuration fails closed', () => { + // Case A: export default calls an undefined function + assert.throws( + () => verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default buildDynamicConfig(); + `, + }, + }), + /Cannot mechanically resolve Playwright configuration/ + ); + + // Case B: export default references an undeclared identifier + assert.throws( + () => verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default nonExistentConfig; + `, + }, + }), + /Cannot mechanically resolve Playwright configuration/ + ); + + // Case C: unimported module / function expression throws runtime error + assert.throws( + () => verifyTestTopology(undefined, { + playwrightConfigOverrides: { + 'packages/web/playwright.config.ts': ` + export default { + testDir: path.join(__dirname, 'e2e'), + }; + `, + }, + }), + /Cannot mechanically resolve Playwright configuration/ + ); +}); + +test('topology: isTestFileReachableByRunner returns false when manifest script is missing', () => { + const missingScript = verifyTestTopology(undefined, { + manifestOverrides: { + 'packages/web/package.json': { + scripts: { + e2e: undefined, + }, + }, + }, + }); + assert.ok(missingScript.unreachable.length >= 25, 'All web e2e tests must be flagged unreachable when e2e script is missing'); + assert.ok(missingScript.unreachable.every((u) => u.suite === 'acceptance-playwright')); +}); + diff --git a/scripts/test/zero-skip-enforcement.test.mjs b/scripts/test/zero-skip-enforcement.test.mjs new file mode 100644 index 00000000..d56e9b62 --- /dev/null +++ b/scripts/test/zero-skip-enforcement.test.mjs @@ -0,0 +1,973 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync, readdirSync } from 'node:fs'; +import { runWithZeroSkip } from '../run-zero-skip.mjs'; +import { selectLiveProviders } from '../run-live-provider-tests.mjs'; +import ZeroSkipReporter, { summarizePlaywrightOutcomes } from '../playwright-zero-skip-reporter.mjs'; +import { runHermeticTests, HERMETIC_WORKSPACES } from '../run-hermetic-tests.mjs'; +import { + createStreamingTestParser, + createStreamLineProcessor, + parseCompleteTestOutput, + parseCancelledCount, + parseFailCount, + parsePassCount, + parseSkippedCount, + parseTestCount, + parseTodoCount, +} from '../lib/test-output-parser.mjs'; + +const NESTED_TAP_SOURCE = 'const { describe, it } = require("node:test"); describe("outer", () => { it("a", () => {}); it("b", () => {}); });'; +const nestedTapEnv = { ...process.env }; +delete nestedTapEnv.NODE_TEST_CONTEXT; + +function actualNestedTap() { + const result = spawnSync(process.execPath, ['--test-reporter=tap', '-e', NESTED_TAP_SOURCE], { encoding: 'utf8', env: nestedTapEnv }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /^ 1\.\.2$/m, 'fixture must contain a nested plan'); + assert.match(result.stdout, /^ok 1 - outer$/m, 'fixture must contain a top-level point'); + assert.match(result.stdout, /^1\.\.1$/m, 'fixture must contain a top-level plan'); + assert.match(result.stdout, /^# tests 2$/m, 'fixture must contain the real Node summary'); + return result.stdout; +} + + +test('zero-skip enforcer: succeeds when a suite reports zero skips', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 5\\n# pass 5\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when skipped is 0'); +}); + +test('zero-skip enforcer: fails closed on an incomplete reporter summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 3\\n# pass 3");', + ], { silent: true }); + assert.equal(code, 1); +}); + +test('zero-skip enforcer: fails closed on contradictory reporter accounting', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 3\\n# pass 3\\n# fail 0\\n# cancelled 0\\n# skipped 1\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1); +}); + +test('zero-skip enforcer: accepts a complete ANSI-colored reporter summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("\\u001b[32m# tests 2\\u001b[0m\\n# pass 2\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0); +}); + +test('zero-skip enforcer: rejects a truncated TAP plan', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - first\\n1..2");', + ], { silent: true }); + assert.equal(code, 1); +}); + +test('zero-skip enforcer: rejects contradictory TAP points even with a complete clean summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - first\\nok 2 - second\\n1..1\\n# tests 2\\n# pass 2\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1); +}); + +test('zero-skip enforcer: accepts ordinary TAP comments resembling metric names', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests initialized\\nok 1 - first\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0); +}); + +test('streaming-stream-processor: oversized unterminated lines fail closed', () => { + const parser = createStreamingTestParser(); + const processor = createStreamLineProcessor(parser); + const chunk = Buffer.alloc(64 * 1024, 120); + for (let i = 0; i < 20; i++) processor.pushStdoutChunk(chunk); + processor.pushStdoutChunk(Buffer.from('\n# tests 1\n# pass 1\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n')); + const result = processor.getResults(); + assert.equal(result.accountingValid, false); + assert.match(result.accountingError, /line exceeds/); +}); + +test('test-output-parser: mixed TAP and summary evidence agrees in a clean run', () => { + const result = parseCompleteTestOutput('ok 1 - first\nok 2 - second\n1..2\n# tests 2\n# pass 2\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0\n'); + assert.equal(result.accountingValid, true); + assert.equal(result.totalTests, 2); +}); + +test('zero-skip enforcer: accepts real Node TAP with nested suites and a complete summary', async () => { + const code = await runWithZeroSkip(process.execPath, ['--test-reporter=tap', '-e', NESTED_TAP_SOURCE], { silent: true, env: nestedTapEnv }); + assert.equal(code, 0); + const parsed = parseCompleteTestOutput(actualNestedTap()); + assert.equal(parsed.accountingValid, true); + assert.equal(parsed.totalTests, 2); + assert.equal(parsed.passCount, 2); +}); + +test('test-output-parser: nested TAP without summary counts the top-level plan only', () => { + const planOnly = actualNestedTap().replace(/^# (?:tests|suites|pass|fail|cancelled|skipped|todo|duration_ms) .*\r?\n/gm, ''); + const parsed = parseCompleteTestOutput(planOnly); + assert.equal(parsed.accountingValid, true); + assert.equal(parsed.totalTests, 1); + assert.equal(parsed.passCount, 1); + assert.equal(parseTestCount(planOnly), 1); +}); + +test('test-output-parser: nested TAP rejects incorrect, duplicate, or missing top-level plans and points', () => { + const tap = actualNestedTap(); + const invalid = [ + tap.replace(/^1\.\.1$/m, '1..2'), + tap.replace(/^1\.\.1$/m, '1..1\n1..1'), + tap.replace(/^1\.\.1\r?\n/m, ''), + tap.replace(/^ok 1 - outer\r?\n/m, ''), + ]; + for (const output of invalid) { + assert.equal(parseCompleteTestOutput(output).accountingValid, false); + } +}); + +test('test-output-parser: nested TAP zero plan cannot be rescued by a positive summary', () => { + const output = actualNestedTap().replace(/^1\.\.1$/m, '1..0'); + const parsed = parseCompleteTestOutput(output); + assert.equal(parsed.totalTests, 0); +}); + +test('test-output-parser: ordinary TAP comments do not corrupt nested summary accounting', () => { + const output = actualNestedTap().replace(/^# tests 2$/m, '# tests initialized\n# tests 2'); + assert.equal(parseCompleteTestOutput(output).accountingValid, true); +}); + +test('test-output-parser: spec summaries ignore ordinary logs resembling TAP prefixes', async () => { + const summary = 'ℹ tests 1\nℹ pass 1\nℹ fail 0\nℹ cancelled 0\nℹ skipped 0\nℹ todo 0'; + for (const log of ['ok: connected', 'not ok: retrying', 'ok (status 200)', 'ok - connected to db', 'ok 200 response sent', 'ok 1 record updated', '1..10 batches processed', ' 1..10 batches processed']) { + const parsed = parseCompleteTestOutput(`${summary}\n${log}`); + assert.equal(parsed.accountingValid, true, log); + assert.equal(parsed.totalTests, 1, log); + } + assert.equal(parseCompleteTestOutput(`${summary}\nnot ok 1 failed`).failCount, 1, 'raw failures still fail under spec summaries'); + assert.equal(parseCompleteTestOutput(`${summary}\nok 1 skipped # SKIP reason`).skippedCount, 1, 'raw skip directives still fail under spec summaries'); + const output = `ok 200 response sent\n${summary}`; + const code = await runWithZeroSkip(process.execPath, ['-e', `console.log(${JSON.stringify(output)})`], { silent: true }); + assert.equal(code, 0, 'the public zero-skip gate accepts a valid spec run with numbered application logs'); +}); + +test('test-output-parser: numbered TAP descriptions need no hyphen and cannot hide skip or failure', async () => { + const clean = parseCompleteTestOutput('ok 1 database query executes\n1..1'); + assert.equal(clean.accountingValid, true); + assert.equal(clean.passCount, 1); + assert.equal(parseCompleteTestOutput('not ok 1 socket timeout\n1..1').failCount, 1); + + const falseCleanSummary = 'ok 1 database query executes # SKIP connection refused\n1..1\n# tests 1\n# pass 1\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0'; + assert.equal(parseCompleteTestOutput(falseCleanSummary).skippedCount, 1); + const code = await runWithZeroSkip(process.execPath, ['-e', `console.log(${JSON.stringify(falseCleanSummary)})`], { silent: true }); + assert.equal(code, 1, 'record-level skip must override a false clean TAP summary'); +}); + +test('test-output-parser: TAP summary still requires a plan after a top-level point', () => { + const output = 'ok 1 test without plan\n# tests 1\n# pass 1\n# fail 0\n# cancelled 0\n# skipped 0\n# todo 0'; + assert.equal(parseCompleteTestOutput(output).accountingValid, false); + assert.equal(parseCompleteTestOutput(output.replaceAll('# ', 'ℹ ')).accountingValid, true, 'spec reporter output may contain numbered application logs'); +}); + +test('test-output-parser: direct streaming accepts CRLF-terminated TAP plan lines', () => { + const parser = createStreamingTestParser(); + parser.pushLine('ok 1 - passes\r'); + parser.pushLine('1..1\r'); + const result = parser.getResults(); + assert.equal(result.accountingValid, true); + assert.equal(result.passCount, 1); + assert.equal(result.totalTests, 1); +}); + +test('test-output-parser: TAP-only passing count excludes failing points', () => { + const parsed = parseCompleteTestOutput('ok 1 - passes\nnot ok 2 - fails\n1..2'); + assert.equal(parsed.accountingValid, true); + assert.equal(parsed.totalTests, 2); + assert.equal(parsed.passCount, 1); + assert.equal(parsed.failCount, 1); +}); + +test('test-output-parser: static TAP fallback rejects duplicate top-level plans', () => { + assert.equal(parseTestCount('ok 1 - first\n1..1\nok 1 - second\n1..1'), null); + assert.equal(parseCompleteTestOutput('ok 1 - first\n1..1\nok 1 - second\n1..1').accountingValid, false); + assert.equal(parseTestCount('ok 1 - first\n1..1 # optional comment'), 1); +}); + +test('live-provider selector supports either credential independently and fails when none exist', () => { + assert.deepEqual(selectLiveProviders('all', { OPENAI_API_KEY: 'present' }), ['openai']); + assert.deepEqual(selectLiveProviders('all', { ANTHROPIC_API_KEY: 'present' }), ['anthropic']); + assert.deepEqual( + selectLiveProviders('all', { OPENAI_API_KEY: 'present', ANTHROPIC_API_KEY: 'present' }), + ['openai', 'anthropic'] + ); + assert.throws(() => selectLiveProviders('all', {}), /at least one/); + assert.throws(() => selectLiveProviders('openai', { ANTHROPIC_API_KEY: 'present' }), /OPENAI_API_KEY/); +}); + +test('live-provider workflow and package scripts execute each provisioned provider independently', () => { + const workflow = readFileSync(new URL('../../.github/workflows/live-provider.yml', import.meta.url), 'utf8'); + assert.match(workflow, /outputs\.has_openai == 'true'/); + assert.match(workflow, /outputs\.has_anthropic == 'true'/); + assert.match(workflow, /test:live-provider:openai --workspace @chess-platform\/ai-orchestrator/); + assert.match(workflow, /test:live-provider:anthropic --workspace @chess-platform\/ai-features/); + assert.doesNotMatch(workflow, /Both OPENAI_API_KEY and ANTHROPIC_API_KEY/); + + for (const packagePath of [ + '../../packages/ai-orchestrator/package.json', + '../../packages/ai-features/package.json', + ]) { + const manifest = JSON.parse(readFileSync(new URL(packagePath, import.meta.url), 'utf8')); + assert.match(manifest.scripts['test:live-provider:openai'], /run-live-provider-tests\.mjs openai/); + assert.match(manifest.scripts['test:live-provider:anthropic'], /run-live-provider-tests\.mjs anthropic/); + } + + for (const testDir of [ + '../../packages/ai-orchestrator/test/', + '../../packages/ai-features/test/', + ]) { + const directory = new URL(testDir, import.meta.url); + for (const entry of readdirSync(directory)) { + if (!entry.endsWith('integration.test.ts')) continue; + const source = readFileSync(new URL(entry, directory), 'utf8'); + assert.doesNotMatch(source, /\bskip\s*:/, `${entry} must not register skipped live tests`); + } + } +}); + +test('Playwright zero-skip accounting rejects skipped, empty, and interrupted suites', () => { + assert.deepEqual(summarizePlaywrightOutcomes(['expected', 'flaky'], 'passed'), { + tests: 2, pass: 2, fail: 0, skipped: 0, todo: 0, cancelled: 0, + }); + assert.equal(summarizePlaywrightOutcomes(['skipped'], 'passed').skipped, 1); + assert.equal(summarizePlaywrightOutcomes(['unexpected'], 'failed').fail, 1); + assert.deepEqual(summarizePlaywrightOutcomes(['expected'], 'interrupted'), { + tests: 1, pass: 0, fail: 0, skipped: 0, todo: 0, cancelled: 1, + }); + assert.deepEqual(summarizePlaywrightOutcomes(['expected'], 'timedout'), { + tests: 1, pass: 0, fail: 1, skipped: 0, todo: 0, cancelled: 0, + }); + assert.equal(summarizePlaywrightOutcomes([], 'passed').tests, 0); +}); + +test('Playwright zero-skip reporter overrides a green run when any test is skipped', () => { + const reporter = new ZeroSkipReporter(); + reporter.onBegin({}, { + allTests: () => [{ outcome: () => 'expected' }, { outcome: () => 'skipped' }], + }); + const originalLog = console.log; + console.log = () => {}; + try { + assert.deepEqual(reporter.onEnd({ status: 'passed' }), { status: 'failed' }); + reporter.onBegin({}, { allTests: () => [{ outcome: () => 'expected' }] }); + assert.deepEqual(reporter.onEnd({ status: 'passed' }), { status: 'passed' }); + reporter.onBegin({}, { allTests: () => [] }); + assert.deepEqual(reporter.onEnd({ status: 'passed' }), { status: 'failed' }); + } finally { + console.log = originalLog; + } +}); + +test('zero-skip enforcer: fails when a suite reports skipped > 0 (TAP format)', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 5\\n# pass 4\\n# skipped 1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when skipped > 0'); +}); + +test('zero-skip enforcer: fails when a suite reports skipped > 0 (Node spec format)', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ℹ tests 10\\nℹ pass 8\\nℹ skipped 2");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when spec format reports skipped > 0'); +}); + +test('zero-skip enforcer: propagates natural non-zero exit code on failure', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'process.exit(42);', + ], { silent: true }); + assert.equal(code, 42, 'should propagate original exit code'); +}); + +test('zero-skip enforcer: detects real child test process self-skipping', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '--input-type=module', + '-e', + 'import test from "node:test"; test("skipping test", { skip: "not provisioned" }, () => {});', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when child test self-skips'); +}); + +test('zero-skip enforcer: does not falsely fail on test names containing the word skipped', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - a stored game whose chess960 metadata is corrupt is skipped, not thrown from\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when test name contains the word skipped'); +}); + +test('zero-skip enforcer: fails when test runner reports 0 tests executed', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 0\\n# pass 0\\n# skipped 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when 0 tests were executed'); +}); + +test('zero-skip enforcer: fails when child process is terminated by a signal', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'process.kill(process.pid, "SIGTERM");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when child process is killed by signal'); +}); + +test('zero-skip enforcer: forwards termination signals and removes temporary handlers', async () => { + const before = new Set(process.listeners('SIGTERM')); + const running = runWithZeroSkip(process.execPath, [ + '-e', + 'setInterval(() => {}, 1000);', + ], { silent: true }); + const handler = process.listeners('SIGTERM').find((listener) => !before.has(listener)); + assert.ok(handler, 'wrapper must install a temporary SIGTERM forwarding handler'); + handler('SIGTERM'); + assert.equal(await running, 1); + assert.deepEqual(new Set(process.listeners('SIGTERM')), before); +}); + +test('zero-skip enforcer: fails when child process exits 0 with arbitrary text and no test summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("Hello world, no tests here");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when arbitrary text without test summary is output'); +}); + +test('zero-skip enforcer: fails unconditionally when a skip is reported', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("not ok 1 - test # SKIP SIGTERM on Windows terminates without running handlers\\n# tests 1\\n# pass 0\\n# skipped 1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when test is skipped regardless of platform'); +}); + +test('zero-skip enforcer: ignores decoy "skipped N" in ordinary output when summary has skipped 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("application skipped 1 old record\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when prose contains "skipped 1" but genuine summary reports skipped 0'); +}); + +test('zero-skip enforcer: fails when summary reports skipped 1 even if ordinary output has decoy "skipped 0"', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("processed item, skipped 0 errors\\n# tests 1\\n# pass 0\\n# skipped 1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when genuine summary reports skipped 1 despite earlier "skipped 0"'); +}); + +test('zero-skip enforcer: ignores decoy "tests 0" in ordinary output when summary has tests 5', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("unrelated tests 0\\n# tests 5\\n# pass 5\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when prose contains "tests 0" but genuine summary reports tests 5'); +}); + +test('zero-skip enforcer: fails when ordinary output contains decoy "tests 5" without real reporter summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("we ran tests 5");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when prose contains "tests 5" but no genuine reporter summary exists'); +}); + +test('zero-skip enforcer: succeeds with valid TAP plan "1..N"', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("1..3\\nok 1 - test a\\nok 2 - test b\\nok 3 - test c");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when genuine TAP plan 1..N is present'); +}); + +test('zero-skip enforcer: fails on decoy "1..N" in ordinary prose without newline anchor', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("see section 1..5 in the manual for details");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when 1..N is unanchored prose'); +}); + +test('zero-skip enforcer: succeeds with valid Node spec reporter output (ℹ tests 5)', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ℹ tests 5\\nℹ pass 5\\nℹ fail 0\\nℹ cancelled 0\\nℹ skipped 0\\nℹ todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 for valid Node spec format summary'); +}); + +test('zero-skip enforcer: fails when an earlier suite in a multi-summary run reports skipped 1 even if later suite reports skipped 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 5\\n# pass 4\\n# skipped 1\\n# tests 5\\n# pass 5\\n# skipped 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when any suite in a multi-summary run reports skipped > 0'); +}); + +test('zero-skip enforcer: fails when an earlier suite in a multi-summary run reports 0 tests executed', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 0\\n# pass 0\\n# skipped 0\\n# tests 5\\n# pass 5\\n# skipped 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when any suite in a multi-summary run executes 0 tests'); +}); + +test('repository-level hermetic runner: fails when a child workspace exits 0 but reports skipped > 0', async () => { + const code = await runHermeticTests({ + workspaces: ['mock-pkg-a', 'mock-pkg-b'], + commandBuilder: (ws) => + ws === 'mock-pkg-b' + ? { cmd: process.execPath, args: ['-e', 'console.log("# tests 1\\n# pass 0\\n# skipped 1");'] } + : { cmd: process.execPath, args: ['-e', 'console.log("# tests 3\\n# pass 3\\n# skipped 0");'] }, + silent: true, + }); + assert.equal(code, 1, 'should fail repository hermetic runner when child workspace reports skipped > 0'); +}); + +test('repository-level hermetic runner: succeeds across multiple workspaces when all report tests > 0 and skipped = 0', async () => { + const code = await runHermeticTests({ + workspaces: ['mock-pkg-a', 'mock-pkg-b', 'mock-pkg-c'], + commandBuilder: () => ({ + cmd: process.execPath, + args: ['-e', 'console.log("ℹ tests 5\\nℹ pass 5\\nℹ fail 0\\nℹ cancelled 0\\nℹ skipped 0\\nℹ todo 0");'], + }), + silent: true, + }); + assert.equal(code, 0, 'should succeed when all child workspaces report valid tests and zero skips'); +}); + +test('repository-level hermetic runner: fails when a child workspace reports zero executed tests', async () => { + const code = await runHermeticTests({ + workspaces: ['mock-pkg-a'], + commandBuilder: () => ({ + cmd: process.execPath, + args: ['-e', 'console.log("# tests 0\\n# pass 0\\n# skipped 0");'], + }), + silent: true, + }); + assert.equal(code, 1, 'should fail when a child workspace executes zero tests'); +}); + +test('repository-level hermetic runner: propagates natural failure exit code when a child workspace fails', async () => { + const code = await runHermeticTests({ + workspaces: ['mock-pkg-a'], + commandBuilder: () => ({ + cmd: process.execPath, + args: ['-e', 'process.exit(42);'], + }), + silent: true, + }); + assert.equal(code, 42, 'should propagate non-zero exit code of failing workspace'); +}); + +test('repository-level hermetic runner: exports the 19 declared hermetic workspaces', () => { + assert.equal(HERMETIC_WORKSPACES.length, 19, 'should declare exactly 19 hermetic workspaces'); + assert.ok(HERMETIC_WORKSPACES.includes('@chess-platform/core')); + assert.ok(HERMETIC_WORKSPACES.includes('@chess-platform/achievements')); + assert.ok(HERMETIC_WORKSPACES.includes('@chess-platform/ai-features')); +}); + +test('zero-skip enforcer: fails when a suite reports TODO-only Node test output (todo > 0)', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 1\\n# pass 0\\n# fail 0\\n# skipped 0\\n# todo 1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when todo > 0 (TODO-only)'); +}); + +test('zero-skip enforcer: fails when a suite reports mixed pass + TODO (todo > 0)', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 2\\n# pass 1\\n# fail 0\\n# skipped 0\\n# todo 1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when mixed pass and todo > 0'); +}); + +test('zero-skip enforcer: succeeds when a suite reports clean Node summary with todo 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 2\\n# pass 2\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when todo is 0'); +}); + +test('zero-skip enforcer: ignores decoy TODO prose in ordinary output when summary has todo 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("todo 5 items remain in application backlog\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should ignore decoy TODO text in application logs'); +}); + +test('zero-skip enforcer: fails when a suite reports cancelled > 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 2\\n# pass 1\\n# fail 0\\n# cancelled 1\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when cancelled > 0'); +}); + +test('zero-skip enforcer: fails when spec reporter format reports todo > 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ℹ tests 3\\nℹ pass 2\\nℹ fail 0\\nℹ cancelled 0\\nℹ skipped 0\\nℹ todo 1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when spec reporter reports todo > 0'); +}); + +test('zero-skip enforcer: fails when spec reporter format reports cancelled > 0', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ℹ tests 3\\nℹ pass 2\\nℹ fail 0\\nℹ cancelled 1\\nℹ skipped 0\\nℹ todo 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when spec reporter reports cancelled > 0'); +}); + +test('zero-skip enforcer: fails when TAP stream contains individual test TODO directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - pending feature # TODO implement next week\\n1..1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when individual test has # TODO'); +}); + +test('zero-skip enforcer: fails when an earlier suite in a multi-summary run reports todo 1', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 3\\n# pass 2\\n# fail 0\\n# skipped 0\\n# todo 1\\n# tests 5\\n# pass 5\\n# fail 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when earlier suite in multi-summary run has todo > 0'); +}); + +test('zero-skip enforcer: fails when an earlier suite in a multi-summary run reports cancelled 1', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 3\\n# pass 2\\n# fail 0\\n# cancelled 1\\n# skipped 0\\n# todo 0\\n# tests 5\\n# pass 5\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when earlier suite in multi-summary run has cancelled > 0'); +}); + +test('zero-skip enforcer: fails when summary reports skipped 0 but individual test has TAP SKIP directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 1\\n# skipped 0\\nok 1 - deferred # SKIP reason");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when TAP # SKIP directive is present despite summary skipped 0'); +}); + +test('zero-skip enforcer: succeeds when summary reports skipped 0 and normal test passes', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - normal\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when clean summary and no skip directive'); +}); + +test('zero-skip enforcer: ignores prose mentioning SKIP reason when summary is clean', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("SKIP reason is logged in prose\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should ignore prose containing SKIP reason'); +}); + +test('zero-skip enforcer: fails when summary reports todo 0 but individual test has TAP TODO directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 1\\n# todo 0\\nok 1 - pending # TODO reason");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when TAP # TODO directive is present despite summary todo 0'); +}); + +test('zero-skip enforcer: succeeds when summary reports todo 0 and test is complete', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - complete\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when clean summary and no todo directive'); +}); + +test('zero-skip enforcer: ignores prose mentioning TODO 3 application tasks when summary is clean', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("TODO 3 application tasks remain in backlog\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should ignore prose containing TODO tasks'); +}); + +test('zero-skip enforcer: fails when raw TAP output has not ok despite child exit 0 and positive plan', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("not ok 1 - failure\\n1..1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when raw TAP contains not ok'); +}); + +test('zero-skip enforcer: succeeds when raw TAP output has ok and positive plan', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - success\\n1..1");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when raw TAP has ok and valid plan'); +}); + +test('zero-skip enforcer: fails when earlier summary reports tests 0 followed by positive tests', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 0\\n# tests 5\\n# pass 5\\n# fail 0\\n# skipped 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when any summary reports tests 0'); +}); + +test('zero-skip enforcer: succeeds when multiple positive summaries are reported', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 2\\n# pass 2\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0\\n# tests 5\\n# pass 5\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when multiple positive summaries pass'); +}); + +test('zero-skip enforcer: fails when per-suite test and pass counts are swapped across summaries', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 2\\n# pass 3\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0\\n# tests 3\\n# pass 2\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1, 'should fail when aggregate totals hide contradictory per-suite accounting'); +}); + +test('zero-skip enforcer: succeeds when distinct per-suite test and pass counts each match', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("# tests 2\\n# pass 2\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0\\n# tests 3\\n# pass 3\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should accept multiple summaries whose accounting is valid suite by suite'); +}); + +test('zero-skip enforcer: fails when TAP plan declares 1..0 despite positive summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("1..0\\n# tests 5\\n# pass 5\\n# skipped 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when TAP plan has 0 tests'); +}); + +test('zero-skip enforcer: fails when unnumbered TAP test point has SKIP directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok - unnumbered test # SKIP not supported\\n1..1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when unnumbered TAP test has # SKIP'); +}); + +test('zero-skip enforcer: fails when unnumbered TAP test point has TODO directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok - unnumbered test # TODO pending feature\\n1..1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when unnumbered TAP test has # TODO'); +}); + +test('zero-skip enforcer: succeeds when test title contains escaped hash before SKIP keyword', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - test title has \\\\# SKIP inside text\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when escaped hash is present in test description'); +}); + +test('test-output-parser: parseTestCount detects zero-test summaries and preserves failure', () => { + assert.equal(parseTestCount('# tests 0\n# tests 5'), 0, '# tests 0 should return 0 even when followed by positive summary'); + assert.equal(parseTestCount('ℹ tests 0\nℹ tests 10'), 0, 'spec format 0 tests should return 0'); + assert.equal(parseTestCount('1..0\n# tests 5'), 0, 'zero-test plan header must return 0 even with positive summary'); + assert.equal(parseTestCount('1..5\n# tests 0'), 0, 'zero-test summary must return 0 even with positive plan header'); + assert.equal(parseTestCount('# tests 2\n# tests 5'), 7, 'multiple positive summaries should aggregate'); + assert.equal(parseTestCount('1..3\nok 1\nok 2\nok 3'), 3, 'TAP-only plan fallback should parse'); + assert.equal(parseTestCount('1..0'), 0, 'TAP-only zero plan should return 0'); + assert.equal(parseTestCount('arbitrary text with no test summary'), null, 'unrecognized output should return null'); +}); + +test('test-output-parser: parseSkippedCount detects TAP and spec skip directives independently of summary', () => { + const mixedOutput = '# tests 1\n# skipped 0\nok 1 - deferred # SKIP temporary reason'; + assert.equal(parseSkippedCount(mixedOutput), 1, 'individual # SKIP should raise count to at least 1 when summary is 0'); + + const cleanOutput = '# tests 1\n# skipped 0\nok 1 - normal'; + assert.equal(parseSkippedCount(cleanOutput), 0, 'clean summary with normal test should be 0'); + + const unnumberedSkip = 'ok - deferred test # SKIP reason\n1..1'; + assert.equal(parseSkippedCount(unnumberedSkip), 1, 'unnumbered ok with # SKIP should be detected'); + + const escapedHashTitle = 'ok 1 - test has \\# SKIP in title\n# tests 1\n# pass 1\n# skipped 0'; + assert.equal(parseSkippedCount(escapedHashTitle), 0, 'escaped hash in test title should not be detected as SKIP'); + + const proseDecoy = 'unrelated log: SKIP reason is handled\n# tests 1\n# skipped 0'; + assert.equal(parseSkippedCount(proseDecoy), 0, 'decoy prose mentioning SKIP reason should be ignored'); + + const tapOnlySkip = 'ok 1 - unsupported platform # SKIP win32 not supported\n1..1'; + assert.equal(parseSkippedCount(tapOnlySkip), 1, 'TAP-only # SKIP without summary should return 1'); +}); + +test('test-output-parser: parseTodoCount detects TAP and spec todo directives independently of summary', () => { + const mixedOutput = '# tests 1\n# todo 0\nok 1 - pending # TODO implement soon'; + assert.equal(parseTodoCount(mixedOutput), 1, 'individual # TODO should raise count to at least 1 when summary is 0'); + + const cleanOutput = '# tests 1\n# todo 0\nok 1 - complete'; + assert.equal(parseTodoCount(cleanOutput), 0, 'clean summary with complete test should be 0'); + + const unnumberedTodo = 'ok - pending test # TODO reason\n1..1'; + assert.equal(parseTodoCount(unnumberedTodo), 1, 'unnumbered ok with # TODO should be detected'); + + const escapedHashTodoTitle = 'ok 1 - test has \\# TODO in title\n# tests 1\n# pass 1\n# todo 0'; + assert.equal(parseTodoCount(escapedHashTodoTitle), 0, 'escaped hash in test title should not be detected as TODO'); + + const proseDecoy = 'TODO 3 application tasks remain in backlog\n# tests 1\n# todo 0'; + assert.equal(parseTodoCount(proseDecoy), 0, 'decoy prose mentioning TODO should be ignored'); + + const tapOnlyTodo = 'ok 1 - deferred feature # TODO not yet ready\n1..1'; + assert.equal(parseTodoCount(tapOnlyTodo), 1, 'TAP-only # TODO without summary should return 1'); +}); + +test('test-output-parser: parseFailCount detects raw TAP not ok and differentiates TODO/SKIP directives', () => { + const rawFailure = 'not ok 1 - broken assertion\n1..1'; + assert.equal(parseFailCount(rawFailure), 1, 'raw TAP not ok should register as failure'); + + const unnumberedRawFailure = 'not ok - broken assertion\n1..1'; + assert.equal(parseFailCount(unnumberedRawFailure), 1, 'unnumbered raw TAP not ok should register as failure'); + + const cleanTap = 'ok 1 - success\n1..1'; + assert.equal(parseFailCount(cleanTap), 0, 'clean TAP ok should not register as failure'); + + const tapTodoNotOk = 'not ok 1 - planned feature # TODO will fix\n1..1'; + assert.equal(parseFailCount(tapTodoNotOk), 0, 'TAP not ok with # TODO should not count as raw failure (handled by todo)'); + assert.equal(parseTodoCount(tapTodoNotOk), 1, 'TAP not ok with # TODO must register as TODO'); + + const tapSkipNotOk = 'not ok 1 - skipped test # SKIP broken environment\n1..1'; + assert.equal(parseFailCount(tapSkipNotOk), 0, 'TAP not ok with # SKIP should not count as raw failure (handled by skip)'); + assert.equal(parseSkippedCount(tapSkipNotOk), 1, 'TAP not ok with # SKIP must register as SKIP'); +}); + +test('zero-skip enforcer: fails when child output contains # SKIPPED directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - feature # SKIPPED not ready\\n# tests 1\\n# pass 1\\n# skipped 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when # SKIPPED is present'); +}); + +test('zero-skip enforcer: fails when child output contains unnumbered # SKIPPED directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok - feature # SKIPPED not ready\\n1..1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when unnumbered # SKIPPED is present'); +}); + +test('zero-skip enforcer: fails when child output contains # TO-DO directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - feature # TO-DO future item\\n# tests 1\\n# pass 1\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when # TO-DO is present'); +}); + +test('zero-skip enforcer: fails when child output contains unnumbered # TO-DO directive', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok - feature # TO-DO future item\\n1..1");', + ], { silent: true }); + assert.equal(code, 1, 'should return exit code 1 when unnumbered # TO-DO is present'); +}); + +test('zero-skip enforcer: succeeds when test title contains escaped hash before SKIPPED keyword', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + 'console.log("ok 1 - title has \\\\# SKIPPED inside\\n1..1\\n# tests 1\\n# pass 1\\n# fail 0\\n# cancelled 0\\n# skipped 0\\n# todo 0");', + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when escaped hash is present in test description'); +}); + +test('test-output-parser: parseSkippedCount and parseTodoCount detect SKIPPED and TO-DO variants', () => { + assert.equal(parseSkippedCount('ok 1 - item # SKIPPED reason\n1..1'), 1); + assert.equal(parseSkippedCount('ok - item # SKIPPED reason\n1..1'), 1); + assert.equal(parseSkippedCount('not ok 1 - item # SKIPPED reason\n1..1'), 1); + assert.equal(parseSkippedCount('ok 1 - item with \\# SKIPPED escaped\n# tests 1\n# skipped 0'), 0); + + assert.equal(parseTodoCount('ok 1 - item # TO-DO reason\n1..1'), 1); + assert.equal(parseTodoCount('ok - item # TO-DO reason\n1..1'), 1); + assert.equal(parseTodoCount('not ok 1 - item # TO-DO reason\n1..1'), 1); + assert.equal(parseTodoCount('ok 1 - item with \\# TO-DO escaped\n# tests 1\n# todo 0'), 0); +}); + +test('streaming-test-parser: maintains O(1) bounded state with identical metrics to static parsers', () => { + const fixtures = [ + '# tests 10\n# pass 8\n# skipped 2\n# fail 0\n# todo 0', + 'ℹ tests 5\nℹ pass 4\nℹ skipped 0\nℹ fail 0\nℹ todo 1', + '1..20\nok 1 - first\nnot ok 2 - fail\nok 3 - skip # SKIP reason', + 'ok 1 - escaped \\# SKIP in title\n# tests 1\n# pass 1\n# skipped 0', + '- test (skipped)\nℹ tests 1\nℹ pass 0\nℹ skipped 0', + '- test (cancelled)\nℹ tests 2\nℹ pass 1\nℹ cancelled 0', + 'not ok 1 - raw unsummarized failure', + '# tests 0\n# pass 0\n# skipped 0', + '1..0', + ]; + + for (const fixture of fixtures) { + const parser = createStreamingTestParser(); + for (const line of fixture.split('\n')) { + parser.pushLine(line); + } + const streamed = parser.getResults(); + + assert.equal(streamed.totalTests, parseTestCount(fixture), `totalTests mismatch on: ${fixture}`); + assert.equal(streamed.skippedCount, parseSkippedCount(fixture), `skippedCount mismatch on: ${fixture}`); + assert.equal(streamed.todoCount, parseTodoCount(fixture), `todoCount mismatch on: ${fixture}`); + assert.equal(streamed.cancelledCount, parseCancelledCount(fixture), `cancelledCount mismatch on: ${fixture}`); + assert.equal(streamed.failCount, parseFailCount(fixture), `failCount mismatch on: ${fixture}`); + } +}); + +test('streaming-test-parser: reconciles many reporter summaries with strictly bounded memory', () => { + const parser = createStreamingTestParser(); + for (let i = 0; i < 20000; i++) { + parser.pushLine('# tests 1'); + parser.pushLine('# pass 1'); + parser.pushLine('# fail 0'); + parser.pushLine('# cancelled 0'); + parser.pushLine('# skipped 0'); + parser.pushLine('# todo 0'); + } + + const results = parser.getResults(); + assert.equal(results.totalTests, 20000); + assert.equal(results.passCount, 20000); + assert.equal(results.skippedCount, 0); + assert.equal(results.failCount, 0); + assert.equal(results.todoCount, 0); + assert.equal(results.cancelledCount, 0); + assert.equal(results.accountingValid, true); +}); + +test('streaming-stream-processor: correctly handles TAP and report lines split across arbitrary chunks', () => { + const parser = createStreamingTestParser(); + const processor = createStreamLineProcessor(parser); + + // Split line across 4 separate chunks + processor.pushStdoutChunk(Buffer.from('# ')); + processor.pushStdoutChunk(Buffer.from('te')); + processor.pushStdoutChunk(Buffer.from('sts ')); + processor.pushStdoutChunk(Buffer.from('12\n')); + + // Split directives across chunks + processor.pushStdoutChunk(Buffer.from('# pass 10\n# skip')); + processor.pushStdoutChunk(Buffer.from('ped 0\n# todo 0\n# cancelled 0\n# fail 2\n')); + + const results = processor.getResults(); + assert.equal(results.totalTests, 12); + assert.equal(results.passCount, 10); + assert.equal(results.skippedCount, 0); + assert.equal(results.failCount, 2); +}); + +test('streaming-stream-processor: prevents cross-stream corruption between interleaved stdout and stderr chunks', () => { + const parser = createStreamingTestParser(); + const processor = createStreamLineProcessor(parser); + + // stdout emits a partial line with a SKIP directive + processor.pushStdoutChunk(Buffer.from('ok 1 - dynamic test # SK')); + + // stderr emits log output in between + processor.pushStderrChunk(Buffer.from('[WARN] connecting to redis replica...\n')); + processor.pushStderrChunk(Buffer.from('[INFO] connected\n')); + + // stdout completes the line + processor.pushStdoutChunk(Buffer.from('IP skipped intentionally\n# tests 1\n# pass 0\n# skipped 1\n')); + + const results = processor.getResults(); + assert.equal(results.totalTests, 1); + assert.equal(results.skippedCount, 1); +}); + +test('streaming-stream-processor: correctly reconstructs multibyte UTF-8 ℹ sequence split across chunk boundaries', () => { + const parser = createStreamingTestParser(); + const processor = createStreamLineProcessor(parser); + + // 'ℹ' is U+2139: UTF-8 bytes 0xE2 0x84 0xB9 + // Chunk 1 has the first 2 bytes: + const chunk1 = Buffer.from([0xe2, 0x84]); + // Chunk 2 has the 3rd byte plus " tests 8\n": + const chunk2 = Buffer.concat([ + Buffer.from([0xb9]), + Buffer.from(' tests 8\nℹ pass 8\nℹ skipped 0\nℹ todo 0\nℹ cancelled 0\nℹ fail 0\n'), + ]); + + processor.pushStdoutChunk(chunk1); + processor.pushStdoutChunk(chunk2); + + const results = processor.getResults(); + assert.equal(results.totalTests, 8, 'totalTests should be 8 from reconstructed ℹ tests 8'); + assert.equal(results.passCount, 8); + assert.equal(results.skippedCount, 0); + assert.equal(results.failCount, 0); +}); + +test('streaming-stream-processor: falsification proves naive chunk.toString and shared lineBuffer fail', () => { + // Falsification Case 1: Naive chunk.toString('utf8') splits multibyte 'ℹ' into replacement characters + const chunk1 = Buffer.from([0xe2, 0x84]); + const chunk2 = Buffer.from([0xb9, 0x20, 0x74, 0x65, 0x73, 0x74, 0x73, 0x20, 0x35, 0x0a]); // " tests 5\n" + const naiveText = chunk1.toString('utf8') + chunk2.toString('utf8'); + assert.ok(naiveText.includes('\ufffd'), 'Naive toString must contain Unicode replacement characters'); + assert.ok(!naiveText.startsWith('ℹ tests 5'), 'Naive toString must fail to reconstruct "ℹ tests 5"'); + + // Falsification Case 2: Shared line buffer between stdout and stderr corrupts anchored TAP directives + let sharedBuffer = ''; + const fakeStdout1 = 'ok 1 - test # SK'; + const fakeStderr = '[LOG] db query error\n'; + const fakeStdout2 = 'IP reason\n'; + + sharedBuffer += fakeStdout1; + sharedBuffer += fakeStderr; + const corruptedLines = sharedBuffer.split('\n'); + assert.equal(corruptedLines[0], 'ok 1 - test # SK[LOG] db query error', 'Shared buffer line was corrupted'); + assert.ok(!corruptedLines[0].includes('# SKIP'), 'Anchored SKIP directive was corrupted into # SK[LOG]'); +}); + +test('zero-skip enforcer: preserves zero-skip pass when child process streams split multibyte UTF-8 summary', async () => { + const code = await runWithZeroSkip(process.execPath, [ + '-e', + ` + // Write first 2 bytes of ℹ (0xE2, 0x84) + process.stdout.write(Buffer.from([0xe2, 0x84])); + // Follow with remaining byte (0xB9) + rest of summary + process.stdout.write(Buffer.concat([ + Buffer.from([0xb9]), + Buffer.from(" tests 4\\nℹ pass 4\\nℹ skipped 0\\nℹ todo 0\\nℹ cancelled 0\\nℹ fail 0\\n") + ])); + `, + ], { silent: true }); + assert.equal(code, 0, 'should return exit code 0 when multibyte ℹ is split across child writes'); +}); + + diff --git a/services/gateway/package.json b/services/gateway/package.json index 3bbfa5dd..18f26280 100644 --- a/services/gateway/package.json +++ b/services/gateway/package.json @@ -8,9 +8,9 @@ "scripts": { "build": "tsc -p tsconfig.json", "start": "node dist/serve.js", - "test": "tsc -p tsconfig.test.json && node --test \"dist-test/test/**/*.test.js\"", - "test:trusted-edge": "npm --prefix ../.. run build:server && npm run build && node --test ../../scripts/nginx-trusted-edge-acceptance.mjs", - "test:web-delivery": "npm --prefix ../.. run build:web && npm --prefix ../.. run build:server && npm run build && node --test ../../scripts/nginx-web-delivery-acceptance.mjs", + "test": "tsc -p tsconfig.test.json && node ../../scripts/run-zero-skip.mjs -- node --test \"dist-test/test/**/*.test.js\"", + "test:trusted-edge": "npm --prefix ../.. run build:server && npm run build && node ../../scripts/run-zero-skip.mjs -- node --test ../../scripts/nginx-trusted-edge-acceptance.mjs", + "test:web-delivery": "npm --prefix ../.. run build:web && npm --prefix ../.. run build:server && npm run build && node ../../scripts/run-zero-skip.mjs -- node --test ../../scripts/nginx-web-delivery-acceptance.mjs", "lint": "tsc -p tsconfig.json --noEmit", "clean": "rm -rf dist dist-test" },