Refresh the realm-server shard weights from CI

.github/workflows/ci-host.yaml

@@ -1473,7 +1473,7 @@ jobs:
             git fetch origin main
 
             if [ "$(git rev-parse origin/main)" != "$WORKFLOW_SHA" ] && \
-               git log --format="%s" "$WORKFLOW_SHA..origin/main" | grep -qvE "Update host test (memory )?baselines?"; then
+               git log --format="%s" "$WORKFLOW_SHA..origin/main" | grep -qvE "Update (host test (memory )?baselines?|matrix shard timings|realm-server shard timings)"; then
               echo "main has advanced past $WORKFLOW_SHA with non-baseline commits — skipping (a newer CI run will publish the baselines)."
               exit 0
             fi

.github/workflows/ci.yaml

@@ -715,7 +715,7 @@ jobs:
 
             if [ "$(git rev-parse origin/main)" != "$WORKFLOW_SHA" ] && \
                git log --format="%s" "$WORKFLOW_SHA..origin/main" \
-                 | grep -qvE "Update (host test (memory )?baselines?|matrix shard timings)"; then
+                 | grep -qvE "Update (host test (memory )?baselines?|matrix shard timings|realm-server shard timings)"; then
               echo "main has advanced past $WORKFLOW_SHA — skipping (a newer CI run will publish the timings)."
               exit 0
             fi
@@ -756,6 +756,10 @@ jobs:
       cancel-in-progress: true
     strategy:
       fail-fast: false
+      # Three places carry the shard count and must move together: this list,
+      # `shardTotal` below, and `--shard-count` on the timings generator in
+      # realm-server-shard-timings-update, which decides whether a rebalance
+      # is worth committing by predicting the slowest shard.
       matrix:
         shardIndex: [1, 2, 3, 4, 5, 6]
         shardTotal: [6]
@@ -940,6 +944,113 @@ jobs:
           path: slot/
           retention-days: 7
 
+  realm-server-shard-timings-update:
+    name: Update Realm Server Shard Timings
+    # Push-to-main only, like its two siblings: this is a job holding
+    # `contents: write`, so it must not run from a ref an untrusted branch can
+    # rewrite.
+    #
+    # Maintains packages/realm-server/tests/test-module-timings.json, the
+    # per-file durations that decide which shard each test file runs on (see
+    # packages/realm-server/scripts/shard-test-modules.cjs). Without this the
+    # weights only change when someone remembers to regenerate them by hand,
+    # and every file added in between packs at DEFAULT_WEIGHT no matter what it
+    # actually costs.
+    #
+    # Runs off the merge job rather than the shards, because that is what
+    # produces the merged report — and it runs whether or not the shards
+    # passed. Files missing from a failed shard's report keep their committed
+    # values, and skipping red runs would leave the weights to rot through a
+    # flaky week.
+    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && !cancelled() && needs.realm-server-merge-reports.result == 'success' }}
+    needs: [realm-server-merge-reports]
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    concurrency:
+      group: realm-server-shard-timings-update-${{ github.head_ref || github.run_id }}
+      cancel-in-progress: true
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          # History past the triggering SHA, so the staleness check below can
+          # see what landed on main while this run was going.
+          fetch-depth: 0
+
+      # mise alone, not `./.github/actions/init`. The generator imports node
+      # builtins and two local scripts and nothing else, so the frozen-lockfile
+      # install that init performs would be several minutes spent on packages
+      # this job never loads. .mise.toml still pins the node version.
+      - uses: jdx/mise-action@c1ecc8f748cd28cdeabf76dab3cccde4ce692fe4 # v4.0.0
+        with:
+          install: true
+
+      - name: Download the merged JUnit report
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: realm-server-test-report-merged
+          path: merged-realm-server-report
+
+      - name: Update shard timings
+        # Retry from the latest main: another PR may merge while this workflow
+        # is running, causing `git push` to be rejected. Each retry fetches and
+        # hard-resets to the freshest tip-of-main, then regenerates against it.
+        #
+        # Staleness guard: if a commit other than an automated measurement
+        # update landed on main since this workflow was triggered, skip — that
+        # newer push triggered its own run, which will publish weights derived
+        # from its own code.
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+          WORKFLOW_SHA="${{ github.sha }}"
+          REPORT="$PWD/merged-realm-server-report/realm-server.xml"
+
+          if [ ! -f "$REPORT" ]; then
+            echo "No merged report at $REPORT — leaving the weights unchanged."
+            exit 0
+          fi
+
+          for attempt in 1 2 3 4 5; do
+            git fetch origin main
+
+            if [ "$(git rev-parse origin/main)" != "$WORKFLOW_SHA" ] && \
+               git log --format="%s" "$WORKFLOW_SHA..origin/main" \
+                 | grep -qvE "Update (host test (memory )?baselines?|matrix shard timings|realm-server shard timings)"; then
+              echo "main has advanced past $WORKFLOW_SHA — skipping (a newer CI run will publish the weights)."
+              exit 0
+            fi
+
+            git reset --hard origin/main
+
+            # Gated on drift: the weights are only rewritten when doing so
+            # improves the predicted slowest shard by over a minute, so
+            # run-to-run jitter doesn't produce a commit per main push. The
+            # shard count must match `shardIndex` on realm-server-test above.
+            #
+            # A nonzero exit — the attribution floor, an unreadable report —
+            # fails the job rather than committing degraded weights.
+            node packages/realm-server/scripts/generate-test-module-timings.mjs "$REPORT" \
+              --min-drift-seconds 60 --shard-count 6
+
+            if git diff --quiet packages/realm-server/tests/test-module-timings.json; then
+              echo "Shard timings unchanged — nothing to commit."
+              exit 0
+            fi
+            git add packages/realm-server/tests/test-module-timings.json
+            git commit -m "Update realm-server shard timings [skip ci]"
+
+            if git push; then
+              exit 0
+            fi
+            echo "Push rejected on attempt $attempt — retrying after $((attempt * 5))s"
+            sleep $((attempt * 5))
+          done
+          echo "Failed to push shard timings after 5 attempts"
+          exit 1
+
   amd-transpile-bench:
     name: AMD Transpile Bench
     needs: change-check

packages/realm-server/scripts/generate-test-module-timings.mjs

@@ -27,16 +27,31 @@
 // Usage:
 //   gh run download <ci-run-id> --name realm-server-test-report-merged -D /tmp/rs
 //   node scripts/generate-test-module-timings.mjs /tmp/rs/realm-server.xml
+//
+// Options:
+//   --min-drift-seconds <n>  Only rewrite the weights when doing so improves
+//                            the predicted slowest shard by at least n
+//                            seconds. Without it every run rewrites the file,
+//                            which on a per-push CI job means a commit per
+//                            push recording nothing but jitter.
+//   --shard-count <n>        The shard count that prediction packs into.
+//                            Required with --min-drift-seconds and with no
+//                            default: "the slowest shard" is meaningless until
+//                            you say how many there are. Keep it equal to the
+//                            realm-server matrix in .github/workflows/ci.yaml.
 
 import { readFileSync, writeFileSync } from 'node:fs';
 import { join, relative } from 'node:path';
 
+import shardTestModules from './shard-test-modules.cjs';
 import {
   createResolver,
   discoverTestFiles,
   testsDir,
 } from './test-module-names.mjs';
 
+const { assignByWeight, weightFor } = shardTestModules;
+
 const timingsPath = join(testsDir, 'test-module-timings.json');
 
 // A report whose suites are mostly unattributed means the reporter regressed
@@ -45,10 +60,43 @@ const timingsPath = join(testsDir, 'test-module-timings.json');
 // so refuse instead.
 const MIN_ATTRIBUTED = 0.9;
 
-const junitPath = process.argv[2];
-if (!junitPath) {
+const args = process.argv.slice(2);
+let junitPath;
+let minDriftSeconds = 0;
+let shardCount;
+for (let i = 0; i < args.length; i++) {
+  if (args[i] === '--min-drift-seconds') {
+    minDriftSeconds = Number(args[++i]);
+  } else if (args[i] === '--shard-count') {
+    shardCount = Number(args[++i]);
+  } else if (!junitPath) {
+    junitPath = args[i];
+  } else {
+    console.error(`Unexpected argument: ${args[i]}`);
+    process.exit(1);
+  }
+}
+
+if (
+  !junitPath ||
+  !Number.isFinite(minDriftSeconds) ||
+  minDriftSeconds < 0 ||
+  (shardCount !== undefined &&
+    !(Number.isInteger(shardCount) && shardCount > 0))
+) {
+  console.error(
+    'Usage: node scripts/generate-test-module-timings.mjs <merged-junit-xml> ' +
+      '[--min-drift-seconds n] [--shard-count n]',
+  );
+  process.exit(1);
+}
+
+// Silently defaulting the shard count would compare slowest-shard times for a
+// suite nobody runs, and the gate would open and close on that fiction.
+if (minDriftSeconds > 0 && shardCount === undefined) {
   console.error(
-    'Usage: node scripts/generate-test-module-timings.mjs <merged-junit-xml>',
+    '--min-drift-seconds requires --shard-count: the prediction is a ' +
+      'slowest-shard time, which depends on how many shards there are.',
   );
   process.exit(1);
 }
@@ -116,11 +164,18 @@ if (coverage < MIN_ATTRIBUTED) {
   process.exit(1);
 }
 
+// Absent is the first run and means there is nothing to preserve. Present but
+// unparseable is a broken file, and swallowing it would drop every weight the
+// current report does not cover — quietly, since the result still looks like a
+// well-formed refresh. shard-test-modules.cjs draws the same distinction.
 let prior = {};
 try {
   prior = JSON.parse(readFileSync(timingsPath, 'utf8'));
-} catch {
-  // First run: nothing to preserve.
+} catch (err) {
+  if (err.code !== 'ENOENT') {
+    console.error(`${timingsPath} exists but could not be read as JSON.`);
+    throw err;
+  }
 }
 
 const merged = {};
@@ -131,14 +186,6 @@ for (const file of onDisk) {
   }
 }
 
-writeFileSync(timingsPath, `${JSON.stringify(merged, null, 2)}\n`);
-
-const fresh = onDisk.filter((f) => timings[f] !== undefined).length;
-const kept = Object.keys(merged).length - fresh;
-console.log(
-  `Wrote ${relative(process.cwd(), timingsPath)}: ${fresh} files measured, ` +
-    `${kept} kept from the previous file, ${(coverage * 100).toFixed(1)}% of ${total.toFixed(0)}s attributed.`,
-);
 if (ambiguous.length) {
   console.warn(
     `Skipped ${ambiguous.length} suite name(s) matching more than one file: ${ambiguous.join(', ')}`,
@@ -147,11 +194,58 @@ if (ambiguous.length) {
 
 // A file that no run has ever measured is packed at DEFAULT_WEIGHT, so a
 // genuinely slow one distorts a shard for as long as it stays invisible. Above
-// the coverage floor that is easy to miss, hence the list.
+// the coverage floor that is easy to miss, hence the list — printed ahead of
+// the drift gate, because the gate asks whether a rebalance is worth a commit,
+// not whether every file has been seen, and the runs it declines are the
+// common case.
 const unmeasured = onDisk.filter((file) => merged[file] === undefined);
 if (unmeasured.length) {
   console.warn(
     `${unmeasured.length} file(s) have no recorded duration and will be packed at the ` +
       `default weight: ${unmeasured.sort().join(', ')}`,
   );
 }
+
+// ---------------------------------------------------------------------------
+// Drift gate. Predict the slowest shard under the committed weights and under
+// the regenerated ones, both scored against the regenerated ones as the better
+// estimate of true duration, and only rewrite if the difference is worth a
+// commit. The packing comes from shard-test-modules.cjs itself rather than a
+// copy of it, so the prediction cannot drift from the assignment it predicts.
+// ---------------------------------------------------------------------------
+
+function slowestShardSeconds(packWeights, trueWeights) {
+  let slowest = 0;
+  for (let shard = 1; shard <= shardCount; shard++) {
+    const load = assignByWeight(onDisk, packWeights, shard, shardCount).reduce(
+      (sum, file) => sum + weightFor(file, trueWeights),
+      0,
+    );
+    slowest = Math.max(slowest, load);
+  }
+  return slowest;
+}
+
+if (minDriftSeconds > 0 && Object.keys(prior).length > 0) {
+  const staleCost = slowestShardSeconds(prior, merged);
+  const freshCost = slowestShardSeconds(merged, merged);
+  const improvement = staleCost - freshCost;
+  console.log(
+    `Predicted slowest shard of ${shardCount}: ${staleCost.toFixed(0)}s with the ` +
+      `committed weights, ${freshCost.toFixed(0)}s with the regenerated ones ` +
+      `(improvement ${improvement.toFixed(0)}s, threshold ${minDriftSeconds}s).`,
+  );
+  if (improvement < minDriftSeconds) {
+    console.log('Within the threshold — leaving the weights unchanged.');
+    process.exit(0);
+  }
+}
+
+writeFileSync(timingsPath, `${JSON.stringify(merged, null, 2)}\n`);
+
+const fresh = onDisk.filter((f) => timings[f] !== undefined).length;
+const kept = Object.keys(merged).length - fresh;
+console.log(
+  `Wrote ${relative(process.cwd(), timingsPath)}: ${fresh} files measured, ` +
+    `${kept} kept from the previous file, ${(coverage * 100).toFixed(1)}% of ${total.toFixed(0)}s attributed.`,
+);