From f81cd0667f3251b6686ba7a2c90bd55a6aecfded Mon Sep 17 00:00:00 2001
From: akacross <jhnbenton@gmail.com>
Date: Sun, 18 Oct 2026 17:28:48 -0400
Subject: [PATCH] Run independent validation checks in parallel

The script-smoke job ran its ~350 checks one after another, about 77 minutes
on main, and every platform job then reran the same Python suite in sequence
before its build (39 of the Linux job's 66 minutes). On the slower runners
that left Windows ARM64 and the macOS Intel jobs cancelled at 150 minutes on
main, and pull-request jobs, with 90-minute limits, cancelled regardless of
the change.

Each check is its own process that writes only to fresh temporary
directories, so they can overlap:

- openq4_validate.py runs its Python checks concurrently, as many as the
  machine has CPUs. --python-jobs N or OPENQ4_PYTHON_TEST_JOBS change that and
  --python-jobs 1 restores sequential runs. Output is still printed in list
  order, one section per check, with each check's duration.
- tools/validation/run_parallel_commands.py does the same for a list of
  commands on standard input; both script-smoke steps feed their unchanged
  check list through it.
- validation_hardening.py covers the parallel path's failure aggregation as
  well as the sequential one.

On a 16-thread Linux host the validator's 350 checks took 2041 s in sequence
and 311 s in parallel with the same results; the whole rewritten smoke step
took 343 s. The only failures, identical either way, are host-toolchain ones
(newer clang/libstdc++), not parallelism.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 .github/workflows/commit-validation.yml   |  5 ++
 .github/workflows/push-verification.yml   |  5 ++
 BUILDING.md                               |  2 +
 tools/tests/validation_hardening.py       | 27 ++++++-
 tools/validation/openq4_validate.py       | 93 ++++++++++++++++++---
 tools/validation/run_parallel_commands.py | 99 +++++++++++++++++++++++
 6 files changed, 220 insertions(+), 11 deletions(-)
 create mode 100644 tools/validation/run_parallel_commands.py

diff --git a/.github/workflows/commit-validation.yml b/.github/workflows/commit-validation.yml
index 89a29ada..a3a29c84 100644
--- a/.github/workflows/commit-validation.yml
+++ b/.github/workflows/commit-validation.yml
@@ -83,6 +83,7 @@ jobs:
             tools/validation/audit_source_provenance.py \
             tools/validation/generate_pbr_fixture.py \
             tools/validation/openq4_validate.py \
+            tools/validation/run_parallel_commands.py \
             tools/validation/stock_asset_baseline.py \
             tools/tests/renderer_validation_matrix.py \
             tools/tests/aas_compiler_contract.py \
@@ -505,6 +506,9 @@ jobs:
           bash -n tools/validation/validate_macos_static.sh
           bash -n tools/validation/validate_push.sh
           bash -n tools/validation/validate_pr.sh
+          # Independent checks run concurrently; run_parallel_commands.py prints each
+          # in this order and fails the step if any fails.
+          python tools/validation/run_parallel_commands.py <<'OPENQ4_CHECKS'
           python tools/validation/audit_source_provenance.py --check
           python tools/tests/aas_compiler_contract.py
           python tools/tests/aas_settings_contract.py
@@ -858,6 +862,7 @@ jobs:
           python tools/tests/vk_display_color_mapping_capture.py --self-test
           python tools/tests/vk_temporal_resolve_shader_pin.py
           python tools/tests/vscode_fast_build.py
+          OPENQ4_CHECKS
 
       - name: Dry-run validation profiles
         shell: bash
diff --git a/.github/workflows/push-verification.yml b/.github/workflows/push-verification.yml
index 7e3bc51b..f70686d8 100644
--- a/.github/workflows/push-verification.yml
+++ b/.github/workflows/push-verification.yml
@@ -83,6 +83,7 @@ jobs:
             tools/validation/audit_source_provenance.py \
             tools/validation/generate_pbr_fixture.py \
             tools/validation/openq4_validate.py \
+            tools/validation/run_parallel_commands.py \
             tools/validation/stock_asset_baseline.py \
             tools/tests/renderer_validation_matrix.py \
             tools/tests/aas_compiler_contract.py \
@@ -505,6 +506,9 @@ jobs:
           bash -n tools/validation/validate_macos_static.sh
           bash -n tools/validation/validate_push.sh
           bash -n tools/validation/validate_pr.sh
+          # Independent checks run concurrently; run_parallel_commands.py prints each
+          # in this order and fails the step if any fails.
+          python tools/validation/run_parallel_commands.py <<'OPENQ4_CHECKS'
           python tools/validation/audit_source_provenance.py --check
           python tools/tests/aas_compiler_contract.py
           python tools/tests/aas_settings_contract.py
@@ -858,6 +862,7 @@ jobs:
           python tools/tests/vk_display_color_mapping_capture.py --self-test
           python tools/tests/vk_temporal_resolve_shader_pin.py
           python tools/tests/vscode_fast_build.py
+          OPENQ4_CHECKS
 
       - name: Dry-run push profile
         shell: bash
diff --git a/BUILDING.md b/BUILDING.md
index 682473d9..84c3b147 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -142,6 +142,8 @@ Pass any of these with `-D<option>=<value>` on the `meson setup` command line:
 
 openQ4 includes local validation profiles under `tools/validation/`. They share one Python runner and use the platform build wrappers so Windows validation still goes through `tools/build/meson_setup.ps1`.
 
+The Python checks are independent and run concurrently, as many at once as the machine has CPUs, with each check's output printed in list order. Pass `--python-jobs N` (or set `OPENQ4_PYTHON_TEST_JOBS`) to change that, and `--python-jobs 1` to run them one at a time. The CI script-smoke steps feed their check list through `tools/validation/run_parallel_commands.py` the same way.
+
 GitHub Actions runs the same validation entrypoints on every pushed commit, pull request, and manual dispatch. Branch pushes run the faster push profile across Linux x64, Linux ARM64, macOS ARM64, macOS Intel x64, Windows x64, and Windows ARM64. Pull requests/manual dispatches run the full PR profile on Windows x64, Windows ARM64, Linux ARM64, and macOS ARM64. Both Windows lanes start the staged dedicated server without game data through `tools/tests/windows_dedicated_server_smoke.py`, but no job starts a Windows client; the staged Linux ARM64 client also runs an assetless no-map renderer startup smoke under a virtual X display. Native Wayland PR jobs run assetless lifecycle, fullscreen/window, input-capture, and display cases under headless Weston on both x64 and ARM64. They verify that preferred libdecor is actually loaded, that the opt-out prevents it from loading, and that the Wayland-only build has no direct X11/GLX dependency. Staged Linux package validation rejects direct PipeWire dependencies from clients and permits only core C/C++/POSIX/compiler runtimes for the x64 and ARM64 dedicated executable plus its loaded MP module. These hosted ARM64 cases prove build/package and window/compositor/input startup, not audio or real SP/MP gameplay.
 
 Manual releases default to `linux_arm64_support_tier=preview`. Selecting `first-class` is a hard gate: the workflow requires accepted physical ARM64 native-Wayland SP/MP, stock-map dedicated-server, audio, input, display, and package evidence in [the Linux ARM64 signoff record](docs/dev/linux-arm64-signoff-evidence.md). The accepted TOML must come from an immutable `linux_arm64_evidence_ref` and match the triggering openQ4 SHA, in-tree game SHA (the same engine commit), release version/tags, exact first-class archive name, archive SHA-256, and all four runtime ELF SHA-256 values. Removing the preview label without that candidate-bound record fails before the release matrix builds.
diff --git a/tools/tests/validation_hardening.py b/tools/tests/validation_hardening.py
index 2c0cd555..81f96849 100644
--- a/tools/tests/validation_hardening.py
+++ b/tools/tests/validation_hardening.py
@@ -586,7 +586,7 @@ def fake_run_command(command, *, cwd, env, title, dry_run) -> None:
     try:
         expect_validation_error(
             lambda: VALIDATOR.run_python_tests(
-                argparse.Namespace(dry_run=False),
+                argparse.Namespace(dry_run=False, python_jobs=1),
                 ROOT,
                 {},
             ),
@@ -596,6 +596,31 @@ def fake_run_command(command, *, cwd, env, title, dry_run) -> None:
     finally:
         VALIDATOR.run_command = original_run_command
 
+    # The parallel runner must aggregate the same failures from both ends.
+    original_captured = VALIDATOR.run_python_check_captured
+    parallel_observed: list[str] = []
+
+    def fake_captured(test_script, test_args, root, env):
+        parallel_observed.append(Path(test_script).name)
+        failed = Path(test_script).name in simulated_failures
+        return (1 if failed else 0), f"simulated {'failure' if failed else 'pass'}\n", 0.0
+
+    VALIDATOR.run_python_check_captured = fake_captured
+    try:
+        expect_validation_error(
+            lambda: VALIDATOR.run_python_tests(
+                argparse.Namespace(dry_run=False, python_jobs=4),
+                ROOT,
+                {},
+            ),
+            "2 Python validation check(s) failed",
+            "aggregated parallel Python validation failures",
+        )
+    finally:
+        VALIDATOR.run_python_check_captured = original_captured
+    if sorted(parallel_observed) != sorted(observed):
+        raise AssertionError("the parallel Python runner did not run the same checks as the sequential one")
+
     if not simulated_failures.issubset(observed):
         raise AssertionError(
             "Python validation stopped before reporting failures from both ends of the suite"
diff --git a/tools/validation/openq4_validate.py b/tools/validation/openq4_validate.py
index b7b756da..9aa2485f 100644
--- a/tools/validation/openq4_validate.py
+++ b/tools/validation/openq4_validate.py
@@ -16,6 +16,7 @@
 import sys
 import time
 import unicodedata
+from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
 
 BUILD_TOOLS_DIR = Path(__file__).resolve().parents[1] / "build"
@@ -403,6 +404,64 @@ def validate_game_libs_repo_path(game_libs_repo: Path) -> Path:
     return game_libs_repo.resolve()
 
 
+def python_test_jobs(args: argparse.Namespace) -> int:
+    requested = getattr(args, "python_jobs", None)
+    if requested is not None:
+        return requested
+    configured = os.environ.get("OPENQ4_PYTHON_TEST_JOBS", "").strip()
+    if configured:
+        return positive_int(configured)
+    return os.cpu_count() or 1
+
+
+def run_python_check_captured(
+    test_script: Path, test_args: list[str], root: Path, env: dict[str, str]
+) -> tuple[int, str, float]:
+    started = time.monotonic()
+    completed = subprocess.run(
+        [sys.executable, str(test_script), *test_args],
+        cwd=str(root),
+        env=env,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+    )
+    return completed.returncode, completed.stdout, time.monotonic() - started
+
+
+def run_python_checks_parallel(
+    checks: list[tuple[Path, list[str]]], root: Path, env: dict[str, str], jobs: int
+) -> list[str]:
+    """Run independent checks concurrently, reporting each in list order.
+
+    Every check is its own interpreter writing only to fresh temporary
+    directories, so they can overlap. Output is held until a check finishes and
+    printed in the original order, so the log reads as a sequential run does.
+    """
+    print(f"\n==> Python checks: {len(checks)} with {jobs} parallel job(s)", flush=True)
+    failures: list[str] = []
+    started = time.monotonic()
+    with ThreadPoolExecutor(max_workers=jobs) as pool:
+        futures = [
+            pool.submit(run_python_check_captured, test_script, test_args, root, env)
+            for test_script, test_args in checks
+        ]
+        for (test_script, test_args), future in zip(checks, futures):
+            title = f"Python check: {rel(test_script, root)}"
+            returncode, output, seconds = future.result()
+            section(title)
+            print(format_command([sys.executable, str(test_script), *test_args]), flush=True)
+            if output:
+                print(output, end="" if output.endswith("\n") else "\n", flush=True)
+            print(f"({seconds:.1f}s)", flush=True)
+            if returncode != 0:
+                failures.append(f"{rel(test_script, root)}: {title} failed with exit code {returncode}.")
+    print(f"\n==> Python checks finished in {time.monotonic() - started:.0f}s", flush=True)
+    return failures
+
+
 def run_python_tests(args: argparse.Namespace, root: Path, env: dict[str, str]) -> None:
     tests = [
         root / "tools" / "tests" / "aas_compiler_contract.py",
@@ -759,20 +818,28 @@ def run_python_tests(args: argparse.Namespace, root: Path, env: dict[str, str])
         (root / "tools" / "tests" / 'vk_display_color_mapping_capture.py', ['--self-test']),
     ]
     failures: list[str] = []
+    checks: list[tuple[Path, list[str]]] = []
     for test_script, test_args in [(path, []) for path in tests] + argument_checks:
         if not test_script.is_file():
             failures.append(f"{rel(test_script, root)}: test script was not found")
             continue
-        try:
-            run_command(
-                [sys.executable, str(test_script), *test_args],
-                cwd=root,
-                env=env,
-                title=f"Python check: {rel(test_script, root)}",
-                dry_run=args.dry_run,
-            )
-        except ValidationError as exc:
-            failures.append(f"{rel(test_script, root)}: {exc}")
+        checks.append((test_script, test_args))
+
+    jobs = python_test_jobs(args)
+    if jobs <= 1 or args.dry_run:
+        for test_script, test_args in checks:
+            try:
+                run_command(
+                    [sys.executable, str(test_script), *test_args],
+                    cwd=root,
+                    env=env,
+                    title=f"Python check: {rel(test_script, root)}",
+                    dry_run=args.dry_run,
+                )
+            except ValidationError as exc:
+                failures.append(f"{rel(test_script, root)}: {exc}")
+    else:
+        failures += run_python_checks_parallel(checks, root, env, jobs)
 
     if failures:
         formatted = "\n".join(f"  - {failure}" for failure in failures)
@@ -1958,6 +2025,12 @@ def parse_args(argv: list[str]) -> argparse.Namespace:
     )
     parser.add_argument("--no-install", dest="install", action="store_false", help="Skip Meson install and staged payload checks.")
     parser.add_argument("--skip-python-tests", action="store_true", help="Skip lightweight Python validation tests.")
+    parser.add_argument(
+        "--python-jobs",
+        type=positive_int,
+        default=None,
+        help="Python validation checks to run at once (default: OPENQ4_PYTHON_TEST_JOBS or the CPU count; 1 runs them in sequence).",
+    )
     parser.add_argument("--skip-build", action="store_true", help="Skip Meson setup/compile/install steps.")
     parser.add_argument("--build-gamelibs", action="store_true", help="Deprecated: game modules now build with the engine.")
     parser.add_argument("--game-libs-repo", default="", help="Override the game-source root for validation fixtures; ordinary builds use this checkout.")
diff --git a/tools/validation/run_parallel_commands.py b/tools/validation/run_parallel_commands.py
new file mode 100644
index 00000000..3498cda1
--- /dev/null
+++ b/tools/validation/run_parallel_commands.py
@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""Run independent validation commands concurrently, reporting them in order.
+
+Reads one command per line from standard input (blank lines and lines starting
+with # are ignored). A leading "python" runs under this interpreter. Commands
+run at once up to --jobs (default: OPENQ4_PYTHON_TEST_JOBS or the CPU count),
+each with its own captured output, which is printed in the input order once that
+command finishes, so the log reads as a sequential run. Exits non-zero when any
+command failed, after listing every failure.
+
+The workflows' script-smoke steps feed their check lists through this. Every
+check runs in its own process and writes only to fresh temporary directories,
+which is what lets them overlap.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import shlex
+import subprocess
+import sys
+import time
+from concurrent.futures import ThreadPoolExecutor
+
+
+def positive_int(value: str) -> int:
+    parsed = int(value)
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError(f"expected a positive integer, got {value!r}")
+    return parsed
+
+
+def default_jobs() -> int:
+    configured = os.environ.get("OPENQ4_PYTHON_TEST_JOBS", "").strip()
+    if configured:
+        return positive_int(configured)
+    return os.cpu_count() or 1
+
+
+def parse_commands(text: str) -> list[list[str]]:
+    commands = []
+    for line in text.splitlines():
+        line = line.strip()
+        if not line or line.startswith("#"):
+            continue
+        argv = shlex.split(line)
+        if argv[0] == "python":
+            argv[0] = sys.executable
+        commands.append(argv)
+    return commands
+
+
+def run_captured(argv: list[str]) -> tuple[int, str, float]:
+    started = time.monotonic()
+    completed = subprocess.run(
+        argv,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+    )
+    return completed.returncode, completed.stdout, time.monotonic() - started
+
+
+def main(argv: list[str]) -> int:
+    parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    parser.add_argument("--jobs", "-j", type=positive_int, default=None, help="Commands to run at once.")
+    args = parser.parse_args(argv)
+    jobs = args.jobs or default_jobs()
+    commands = parse_commands(sys.stdin.read())
+
+    print(f"==> {len(commands)} validation command(s) with {jobs} parallel job(s)", flush=True)
+    started = time.monotonic()
+    failures: list[str] = []
+    with ThreadPoolExecutor(max_workers=jobs) as pool:
+        futures = [pool.submit(run_captured, command) for command in commands]
+        for command, future in zip(commands, futures):
+            shown = shlex.join(["python", *command[1:]] if command[0] == sys.executable else command)
+            returncode, output, seconds = future.result()
+            print(f"\n==> {shown}", flush=True)
+            if output:
+                print(output, end="" if output.endswith("\n") else "\n", flush=True)
+            print(f"({seconds:.1f}s, exit {returncode})", flush=True)
+            if returncode != 0:
+                failures.append(f"{shown} (exit {returncode})")
+
+    print(f"\n==> Finished {len(commands)} command(s) in {time.monotonic() - started:.0f}s", flush=True)
+    if failures:
+        print(f"{len(failures)} validation command(s) failed:", flush=True)
+        for failure in failures:
+            print(f"  - {failure}", flush=True)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main(sys.argv[1:]))