Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
622c8f8
ci: enforce strict zero-test-skip architecture and environment suite …
sayed710 Sep 16, 2026
f03a841
ci: refine zero-skip summary parser and migrate test database in back…
sayed710 Sep 16, 2026
903d400
ci: address review findings for signal handling, test compilation, an…
sayed710 Sep 16, 2026
27c3c66
ci: close pool before backup-restore drill to prevent connection term…
sayed710 Sep 16, 2026
c3e3572
ci: enforce positive test output, partition posix suites, self-contai…
sayed710 Sep 16, 2026
27712ec
fix(ci): harden backup-restore drill against teardown error and remov…
sayed710 Sep 17, 2026
77767f3
ci: anchor test-count and skip-count parsers to genuine reporter lines
sayed710 Sep 17, 2026
f43d350
ci: aggregate test counts and skips across multi-summary test outputs
sayed710 Sep 17, 2026
4b67b12
ci: enforce centralized repository-wide hermetic zero-skip orchestrat…
sayed710 Sep 17, 2026
5ebeda5
ci: reject TODO and cancelled test metrics across zero-skip enforcer …
sayed710 Sep 17, 2026
979d9b8
ci: reconcile TAP directives, raw failures, and per-summary zero-test…
sayed710 Sep 17, 2026
81e08de
ci: support unnumbered TAP directives and harden against escaped hash…
sayed710 Sep 17, 2026
40d3be2
ci: enforce universal discovery, runner reachability, and stream buff…
sayed710 Sep 20, 2026
b6bc8c7
ci: mechanically derive runner reachability from manifests and bound …
sayed710 Sep 21, 2026
e123c76
ci: separate stream state in runner, validate playwright config mecha…
sayed710 Sep 21, 2026
6374a1a
fix(scripts): handle **/ as zero or more path segments in fallback gl…
sayed710 Sep 21, 2026
57a2d83
fix(topology): fail closed on unparseable or non-literal Playwright t…
sayed710 Sep 21, 2026
0555ebe
fix(topology): restrict Playwright testDir and testMatch extraction t…
sayed710 Sep 21, 2026
bb41a55
fix(topology): derive Playwright reachability via authoritative test …
sayed710 Sep 21, 2026
6228891
fix(topology): guard scriptCmd before checking runner type (CodeRabbit)
sayed710 Sep 21, 2026
a085542
merge: integrate current main into PR 57
sayed710 Sep 22, 2026
9ca26a9
fix(ci): close strict zero-skip enforcement gaps
sayed710 Sep 22, 2026
d12180a
docs(test): correct adapter suite guidance
sayed710 Sep 22, 2026
39b46d9
fix: enforce per-suite zero-skip accounting
sayed710 Sep 22, 2026
5ac8200
fix(ci): close zero-skip parser and offline e2e gaps
sayed710 Sep 22, 2026
dc9f764
fix(ci): reconcile nested TAP against top-level plan
sayed710 Sep 22, 2026
0610ef2
fix(ci): recognize TAP descriptions and refresh Anthropic test model
sayed710 Sep 23, 2026
4a80927
fix(ci): distinguish spec application logs from TAP plans
sayed710 Sep 23, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,12 @@ jobs:
- name: Load harness contract tests
run: npm run test:load-harness

- name: POSIX API tests
run: npm run test:posix -w @chess-platform/api

- name: POSIX load harness tests
run: npm run test:load-harness:posix

# The container images build with their own package chain, which CI never exercises — it
# builds from the root chain instead. That gap let `docker compose up --build` stay broken
# from M11 inc 5 onwards while every gate here was green. Static, so it costs seconds.
Expand Down Expand Up @@ -199,6 +205,10 @@ jobs:
# stopped looking at anything. These drive it against synthetic sources: a commented-out
# entry, a forward migration, a constraint replaced rather than edited, a renamed
# declaration. Pure functions over temp files, no services.
# Every test file in the repository must belong to an explicit, appropriately provisioned suite.
- name: Test topology check
run: npm run check:test-topology

- name: Guard script tests
run: npm run test:scripts

Expand Down Expand Up @@ -403,10 +413,13 @@ jobs:
run: npm run build

- name: Test persistence against Postgres
run: npm test --workspace @chess-platform/persistence
run: npm run test:integration:postgres --workspace @chess-platform/persistence

- name: Test API concurrency controls against Postgres
run: npm test --workspace @chess-platform/api
run: npm run test:integration:postgres --workspace @chess-platform/api
Comment thread
qodo-code-review[bot] marked this conversation as resolved.
Comment thread
coderabbitai[bot] marked this conversation as resolved.

- name: Test scripts integration against Postgres
run: npm run test:scripts:integration

# The only job that runs a real engine binary (ADR-0113). Every other analysis test drives a
# fake transport or a provider double, which keeps the main suite hermetic but leaves the
Expand Down
75 changes: 75 additions & 0 deletions .github/workflows/live-provider.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
name: Live Provider Contract Tests

# Dedicated, manual-only workflow for live third-party AI provider contract tests.
# These tests make real network calls to OpenAI / Anthropic APIs and require live API keys.
# Per repository policy:
# - They MUST NOT run in automatic PR CI.
# - They MUST NOT be marked as passed when credentials are unavailable.
# - They execute strictly on demand when credentials are provided in repository secrets.

on:
workflow_dispatch:

permissions:
contents: read

jobs:
live-provider-contract:
name: live provider contract tests (provisioned providers)
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v4

- name: Set up Node 22.x
uses: actions/setup-node@v4
with:
node-version: '22.x'
cache: npm

- name: Install dependencies (reproducible)
run: npm ci

- name: Build (dependency order)
run: npm run build

- name: Detect provisioned credentials
id: credentials
run: |
has_openai=false
has_anthropic=false
if [ -n "${OPENAI_API_KEY}" ]; then has_openai=true; fi
if [ -n "${ANTHROPIC_API_KEY}" ]; then has_anthropic=true; fi
echo "has_openai=${has_openai}" >> "$GITHUB_OUTPUT"
echo "has_anthropic=${has_anthropic}" >> "$GITHUB_OUTPUT"
if [ "${has_openai}" = false ] && [ "${has_anthropic}" = false ]; then
echo "::error::At least one live-provider credential must be configured."
exit 1
fi
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

- name: Test AI orchestrator OpenAI contract
if: steps.credentials.outputs.has_openai == 'true'
run: npm run test:live-provider:openai --workspace @chess-platform/ai-orchestrator
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

- name: Test AI orchestrator Anthropic contract
if: steps.credentials.outputs.has_anthropic == 'true'
run: npm run test:live-provider:anthropic --workspace @chess-platform/ai-orchestrator
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

- name: Test AI features OpenAI contract
if: steps.credentials.outputs.has_openai == 'true'
run: npm run test:live-provider:openai --workspace @chess-platform/ai-features
env:
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

- name: Test AI features Anthropic contract
if: steps.credentials.outputs.has_anthropic == 'true'
run: npm run test:live-provider:anthropic --workspace @chess-platform/ai-features
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
46 changes: 46 additions & 0 deletions deploy/load/test/run-evidence.posix.test.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
import { test } from 'node:test';
import assert from 'node:assert/strict';
import { spawn } from 'node:child_process';
import { once } from 'node:events';
import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
import { tmpdir } from 'node:os';
import { dirname, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';

const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..', '..');

const armScript = (dir, body, options = 'undefined') => `
import { existsSync, writeFileSync } from 'node:fs';
import { armFailureEvidence, buildEvidence, clearEvidence, writeEvidence } from ${JSON.stringify(
pathToFileURL(join(REPO_ROOT, 'scripts/lib/run-evidence.mjs')).href,
)};
const DIR = ${JSON.stringify(dir)};
const FILE = 'evidence.json';
const build = (exitCode) => buildEvidence({
harness: 'test', outcome: 'aborted', exitCode,
startedAt: '2026-08-27T10:00:00.000Z', finishedAt: '2026-08-27T10:00:01.000Z',
});
clearEvidence(DIR, FILE);
const fallback = armFailureEvidence(DIR, FILE, build, ${options});
${body}
`;

test('a real SIGTERM leaves the artifact and still terminates by the signal', async () => {
const dir = mkdtempSync(join(tmpdir(), 'gambit-evidence-'));
const file = join(dir, 'child.mjs');
writeFileSync(file, armScript(dir, "console.log('armed');\nsetTimeout(() => {}, 60_000);"), 'utf8');

const child = spawn(process.execPath, [file], { cwd: REPO_ROOT, encoding: 'utf8' });
await once(child.stdout, 'data');
child.kill('SIGTERM');
const [code, signal] = await once(child, 'exit');

assert.equal(
signal,
'SIGTERM',
'the handler re-raises after writing, so the process still dies BY the signal rather than ' +
'turning an interrupt into an ordinary exit',
);
assert.equal(code, null);
assert.equal(JSON.parse(readFileSync(join(dir, 'evidence.json'), 'utf8')).exitCode, 143);
});
24 changes: 0 additions & 24 deletions deploy/load/test/run-evidence.test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -413,30 +413,6 @@ test('an interrupt after clearing still leaves a failure artifact', () => {
assert.notEqual(result.status, 0, 'and an interrupted run must never look like a successful one');
});

test(
'a real SIGTERM leaves the artifact and still terminates by the signal',
{ skip: process.platform === 'win32' ? 'Windows terminates on kill() without running handlers' : false },
async () => {
const dir = mkdtempSync(join(tmpdir(), 'gambit-evidence-'));
const file = join(dir, 'child.mjs');
writeFileSync(file, armScript(dir, "console.log('armed');\nsetTimeout(() => {}, 60_000);"), 'utf8');

const child = spawn(process.execPath, [file], { cwd: REPO_ROOT, encoding: 'utf8' });
await once(child.stdout, 'data');
child.kill('SIGTERM');
const [code, signal] = await once(child, 'exit');

assert.equal(
signal,
'SIGTERM',
'the handler re-raises after writing, so the process still dies BY the signal rather than ' +
'turning an interrupt into an ordinary exit',
);
assert.equal(code, null);
assert.equal(JSON.parse(readFileSync(join(dir, 'evidence.json'), 'utf8')).exitCode, 143);
},
);

test('a run that wrote its own evidence is not overwritten by the fallback', () => {
const dir = mkdtempSync(join(tmpdir(), 'gambit-evidence-'));
const result = runInChild(
Expand Down
Loading
Loading