diff --git a/.github/workflows/bf-pr-context.yml b/.github/workflows/bf-pr-context.yml new file mode 100644 index 0000000000000..f797cfd5cfd24 --- /dev/null +++ b/.github/workflows/bf-pr-context.yml @@ -0,0 +1,252 @@ +name: "BF: PR Context CI" + +# Assembles the REPRESENTATIVE tree for patch-branch PRs — the same tree +# bf-build-stable would build on merge: upstream base + all manifest +# patches (at their recorded SHAs) + this PR's head — and runs the +# upstream test suite against THAT, instead of the raw base+one-patch +# tree that upstream PR CI sees. +# +# Why: PRs against brightfire/ inherit a background failure set +# from the missing patch context (other patches' skips and fixes don't +# apply), which buries the signal ("did MY change break this?"). A +# failure on the assembled tree means "stable will fail if this merges". +# +# Raw-tree upstream CI still runs on patch PRs and catches patch-local +# hygiene (lint, knip, max-lines). This workflow answers the integration +# question. Green means green: any failing test on the assembled tree +# gates the PR. Nothing is ledgered or exempted. See docs/brightfire-ci.md. + +on: + pull_request: + types: [opened, synchronize, reopened, closed] + branches: + - "brightfire/**" + - "!brightfire/ci" + +concurrency: + group: pr-context-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: write + +env: + PATCHES_FILE: BRIGHTFIRE_PATCHES.md + PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false" + +jobs: + # ======================================================================== + # Assemble: base + all manifest patches (recorded SHAs) + PR head, + # regenerate artifacts, compute the test matrix, push to a temp ref. + # ======================================================================== + assemble: + name: Assemble PR context tree + if: >- + github.event_name == 'pull_request' && + github.event.action != 'closed' && + github.event.pull_request.base.ref != 'brightfire/ci' + runs-on: ubuntu-24.04 + timeout-minutes: 30 + outputs: + context_ref: ${{ steps.push.outputs.ref }} + test_shards: ${{ steps.test-shards.outputs.shards }} + steps: + - name: Checkout brightfire/ci + uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - name: Parse manifest (patches + base branch) + id: patches + run: | + python3 scripts/bf/parse-patches.py "$PATCHES_FILE" "$GITHUB_OUTPUT" + + - name: Read upstream version from manifest + id: upstream-version + run: | + VERSION=$(grep -oP '(?<=Upstream version:\*\*\s\x60v)[^\x60]+' "$PATCHES_FILE") + if [ -z "$VERSION" ]; then + echo "::error::Could not read Upstream version from $PATCHES_FILE _meta" + exit 1 + fi + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + # Preserve every CI-side input from this exact brightfire/ci revision + # before switching to the assembled tree (same pattern as + # bf-build-stable). + - name: Preserve Brightfire CI inputs + run: | + INPUTS=/tmp/bf-ci-inputs + mkdir -p "$INPUTS/scripts" + cp "$PATCHES_FILE" "$INPUTS/BRIGHTFIRE_PATCHES.md" + cp -a scripts/bf "$INPUTS/scripts/" + echo "BF_CI_INPUTS_DIR=$INPUTS" >> "$GITHUB_ENV" + echo "PATCHES_FILE=$INPUTS/BRIGHTFIRE_PATCHES.md" >> "$GITHUB_ENV" + + - name: Fetch patch branches and PR head + run: | + git fetch origin '+refs/heads/brightfire/*:refs/remotes/origin/brightfire/*' + git fetch origin "${{ github.event.pull_request.head.sha }}" + + - name: Prepare base checkout + env: + BASE_COMMIT: ${{ steps.patches.outputs.base_commit }} + run: | + git config user.name "brightfire-ci" + git config user.email "ci@brightfire.net" + git checkout -B pr-context-work "$BASE_COMMIT" + + # Merge every patch at its RECORDED manifest SHA (the contract), + # then the PR head on top — exactly what the next build-stable + # would assemble after this PR merges and register-patch records it. + - name: Merge patches in manifest order + env: + PATCHES: ${{ steps.patches.outputs.list }} + VERSION: ${{ steps.upstream-version.outputs.version }} + run: /tmp/bf-ci-inputs/scripts/bf/build-stable/merge-patches.sh + + - name: Merge PR head + run: | + git merge --no-edit "${{ github.event.pull_request.head.sha }}" + echo "Assembled: $(git rev-parse HEAD)" + + - name: Setup Node environment + uses: ./.github/actions/setup-node-env + with: + node-version: "24" + install-bun: "false" + cache-mode: "restore" + install-deps: "false" + + - name: Install dependencies + run: pnpm install --no-frozen-lockfile + + - name: Regenerate build artifacts + run: /tmp/bf-ci-inputs/scripts/bf/build-stable/regenerate-artifacts.sh + + - name: Compute test matrix + id: test-shards + run: | + node --import tsx -e " + const plan = await import('./scripts/lib/ci-node-test-plan.mts'); + const create = typeof plan.createNodeTestShardBundles === 'function' + ? plan.createNodeTestShardBundles + : plan.createNodeTestShards; + const shards = await create({ runnerBackend: 'github' }); + const runnerMap = { + 'blacksmith-4vcpu-ubuntu-2404': 'ubuntu-24.04', + 'blacksmith-8vcpu-ubuntu-2404': 'ubuntu-24.04', + 'blacksmith-16vcpu-ubuntu-2404': 'ubuntu-24.04', + }; + const matrix = shards.map(s => ({ + shard_name: s.shardName ?? s.checkName, + groups: s.groups ?? null, + configs: s.configs ?? [], + runner: runnerMap[s.runner] ?? s.runner ?? 'ubuntu-24.04', + timeout_minutes: s.timeoutMinutes ?? 60, + env: s.env ?? {}, + includePatterns: s.includePatterns ?? [], + targets: s.targets ?? [], + plan_concurrency: s.planConcurrency ?? null, + pretest_build_mode: s.pretestBuildMode ?? null, + requires_dist: s.requiresDist ?? false, + requires_go: s.requiresGo ?? false, + requires_ripgrep: s.requiresRipgrep ?? false, + })); + console.log(JSON.stringify(matrix)); + " > /tmp/test-matrix.json + SHARDS=$(cat /tmp/test-matrix.json) + echo "shards=$SHARDS" >> "$GITHUB_OUTPUT" + SHARD_COUNT=$(echo "$SHARDS" | python3 -c "import json,sys; print(len(json.load(sys.stdin)))") + echo "Test shards ($SHARD_COUNT)" + + # GITHUB_TOKEN push: does not trigger downstream workflows (fine — + # nothing triggers on pr-context/*), and branch protection doesn't + # cover pr-context/* refs. Force-push overwrites stale assemblies. + - name: Push context ref + id: push + run: | + REF="pr-context/${{ github.event.pull_request.number }}" + git push --force origin "HEAD:refs/heads/$REF" + echo "ref=$REF" >> "$GITHUB_OUTPUT" + + # ======================================================================== + # Test: the extracted suite against the assembled ref. cache-mode stays + # at its default (restore): PR runs never write the dist cache — warming + # release caches is the release lane's job. + # ======================================================================== + test: + name: Run upstream CI test suite (assembled tree) + needs: assemble + uses: ./.github/workflows/ci-test-suite.yml + with: + test_ref: ${{ needs.assemble.outputs.context_ref }} + test_matrix: ${{ needs.assemble.outputs.test_shards }} + + # ======================================================================== + # Summary: any failing test on the assembled tree gates the PR. Green + # means green — no ledger, no exemptions. + # ======================================================================== + summary: + name: Report assembled-tree result + needs: [assemble, test] + if: always() && needs.assemble.result == 'success' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + env: + GITHUB_TOKEN: ${{ github.token }} + steps: + - name: Collect failing test files + id: failures + run: | + mkdir -p /tmp/prctx + FAILED_FILES="$(mktemp)" + for job_id in $(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs?per_page=200" --paginate --jq '.jobs[] | select(.conclusion == "FAILURE" and (.name | startswith("Run upstream CI test suite"))) | .databaseId'); do + gh api "repos/$GITHUB_REPOSITORY/actions/jobs/$job_id/logs" 2>/dev/null \ + | awk 'length($0) < 400' \ + | grep -oE '× +[a-z-]+ +[^ ]+\.test\.[a-z]+' \ + | awk '{print $NF}' >> "$FAILED_FILES" || true + done + sort -u "$FAILED_FILES" > /tmp/prctx/actual-failures.txt || true + ACTUAL_COUNT=$(wc -l < /tmp/prctx/actual-failures.txt) + echo "actual_count=$ACTUAL_COUNT" >> "$GITHUB_OUTPUT" + cat /tmp/prctx/actual-failures.txt + + - name: Comment on PR + run: | + PR="${{ github.event.pull_request.number }}" + ACTUAL_COUNT="${{ steps.failures.outputs.actual_count }}" + if [ "$ACTUAL_COUNT" -gt 0 ]; then + FAIL_LIST=$(sed 's/^/- /' /tmp/prctx/actual-failures.txt) + gh pr comment "$PR" --body "$(cat < ref when the PR closes. + # ======================================================================== + cleanup: + name: Delete context ref + if: github.event.action == 'closed' + runs-on: ubuntu-24.04 + timeout-minutes: 5 + env: + GITHUB_TOKEN: ${{ github.token }} + steps: + - name: Delete pr-context ref + run: | + REF="heads/pr-context/${{ github.event.pull_request.number }}" + STATUS=$(gh api -o /dev/null -w "%{http_code}" "repos/$GITHUB_REPOSITORY/git/$REF" 2>/dev/null || echo 404) + if [ "$STATUS" = "200" ]; then + gh api -X DELETE "repos/$GITHUB_REPOSITORY/git/$REF" + echo "Deleted $REF" + else + echo "No context ref to delete" + fi diff --git a/docs/brightfire-ci.md b/docs/brightfire-ci.md new file mode 100644 index 0000000000000..64a3c1e779945 --- /dev/null +++ b/docs/brightfire-ci.md @@ -0,0 +1,58 @@ +# Brightfire CI Guide + +How CI works on the Brightfire fork, how to read a red PR, and where test changes belong. + +## The two questions CI answers + +A patch PR gets two kinds of signal: + +1. **"Is my patch clean on its own?"** — upstream CI runs against the raw PR tree (base + this patch only). Catches patch-local hygiene: lint, knip, max-lines, formatting. Runs on + every PR; failures here are usually about the patch itself. +2. **"Will the integration work?"** — `bf-pr-context.yml` assembles the representative tree (upstream base + ALL manifest patches at their recorded SHAs + the PR head — exactly + what `bf-build-stable` would build on merge) and runs the upstream test suite against that. A failure here means stable will fail if this merges. + +The raw-tree checks will show failures from _missing patch context_ (see taxonomy below) — those are noise on question 1 and are the reason question 2 exists. The PR-context +summary gates on any failing test: green means green, with no known-failure exemption list. A failure that exists independent of the PR under review gets fixed or +skipped at the proper layer (see taxonomy), never bookkept. + +## Failure taxonomy + +Every red check is one of: + +| Class | Example | Where the fix goes | +| ------------------------ | ------------------------------------------------------ | ------------------------------------------------------------- | +| PR's own change | a test your patch actually breaks; a guard it violates | the PR itself | +| Patch contract change | upstream test asserts the old contract the patch flips | the owning patch, per the doctrine below | +| Pre-existing at base | fails on the clean upstream tree too | `upstream-test-fixes` (skip, with clean-base evidence) | +| CI-environment-sensitive | passes locally, fails on GH-hosted runners | `upstream-test-fixes` (skip, with CI-run evidence) | +| Missing patch context | fails only on raw-tree PR CI — other patches missing | nowhere — PR-context CI suppresses it; do not "fix" it per-PR | +| Infra flake | artifact-service 403, runner failures | rerun; do not code around | + +## Test-change doctrine (per-assertion) + +When a patch deliberately changes a contract (e.g. bundle-all-plugins ships all plugins in the tarball), evaluate failing tests **per assertion**, not per file: + +1. The whole case asserts the old contract (externality/exclusion) → invalid on the patched tree → skip the case (or delete it if the patch erroneously carried it). +2. The old contract is one condition inside a broader valid test → surgically remove that condition; keep the test active. +3. The test asserts presence/correctness of what the patch now ships → update its expectations to the patched tree's truth; the test now **enforces** the new contract. +4. Ambiguous → skip, and say why in the resolution note. + +**Skip placement:** env/pre-existing failures go in `upstream-test-fixes` (applies first, so every later tree inherits the green baseline). Contract changes go in the patch that +changes the contract. CI infrastructure goes on `brightfire/ci`. Never maintain the same skip in two places — if you find yourself copying a skip between the patch stack and a +branch, the assembly step is missing (see PR-context CI). + +**Before skipping anything because "the patch changed the contract":** check what the test actually pins. A test that fails because of a patch can still be flagging a real bug +**in** the patch (example: the `cli-http-fallback` probe test caught the /ready fallback masking explicitly rejected tokens — the fix was gating the patch, not skipping the test). + +## Triage playbook + +1. **Never reason from absolute red/green.** Diff the PR's failing checks against the base branch's last CI run, or against the previous head's run. New-vs-known is the only + signal. +2. For each new failing test, classify it against the taxonomy above. Reproduce on the clean base before calling anything "pre-existing". +3. Route the fix per the table. Patch PRs should not accumulate skips for classes that PR-context CI already covers. + +## Merge order and intermediate red builds + +`bf-register-patch` commits manifest bumps to `brightfire/ci` on every patch-PR merge, and those commits touch `BRIGHTFIRE_PATCHES.md` — which triggers `bf-build-stable`. Multi-PR +sets therefore produce **intermediate red build-stable runs** while the set is landing. That is expected: use those runs as incremental verification of what has landed so far, and +expect the final PR in the set (usually the `brightfire/ci` one) to trigger the first fully green run.