replay: overlap execution and improve checkpoints and epoch recovery

.github/workflows/go_build.yml

@@ -35,13 +35,15 @@ jobs:
 
       - name: Voting, checkpoint, streaming and scheduler race regressions
         # Run the complete affected package suites, including subprocess crash
-        # recovery and cancellation tests. The independent sealevel suite has
-        # known base-branch failures documented in the validation report.
+        # recovery and cancellation tests.
         run: >-
           go test -race -p 2 -count=1
-          ./pkg/alpenglow ./pkg/consensus ./pkg/replay
+          ./pkg/alpenglow ./pkg/consensus ./pkg/replay ./pkg/rewards
           ./pkg/turbine ./pkg/sigverify ./pkg/blockprod/...
           ./cmd/mithril/node ./cmd/mithril/configcmd
 
-      - name: Vote-program deque ownership race regression
-        run: go test -race -count=1 ./pkg/sealevel -run '^TestProcessNewVoteStateOwnsRetainedDeque$'
+      - name: Interpreter differential and memory race regressions
+        run: go test -race -count=1 ./pkg/sbpf/...
+
+      - name: Sealevel interpreter, syscall and vote ownership regressions
+        run: go test -race -count=1 ./pkg/sealevel

cmd/mithril/configcmd/configcmd.go

@@ -158,6 +158,9 @@ authorized_voter_keypair = ""      # BLS derivation signer (empty defaults to id
 authorized_withdrawer_keypair = "" # Authorized withdrawer keypair path (diagnostics only)
 tpu_quic_bind_addr = "0.0.0.0:8004"
 advertised_ip = ""                 # Required only in validator mode; public IP advertised for TPU QUIC
+wait_to_vote_slot = 0             # Minimum slot for new votes; does not bypass recovery checks
+tpu_max_buffered_transactions = 0 # 0 = 131,072 buffered transactions
+block_completion_reserve_ms = 0   # 0 = 75ms local completion/broadcast reserve
 tpu_sigverify_workers = 0          # 0 = GOMAXPROCS
 
 [consensus]
@@ -175,6 +178,9 @@ authorized_voter_keypair = ""                               # Empty defaults to
 authorized_withdrawer_keypair = ""
 tpu_quic_bind_addr = "0.0.0.0:8004"
 advertised_ip = "" # REQUIRED: public IP advertised for TPU QUIC
+wait_to_vote_slot = 0             # Minimum slot for new votes; does not bypass recovery checks
+tpu_max_buffered_transactions = 0 # 0 = 131,072 buffered transactions
+block_completion_reserve_ms = 0   # 0 = 75ms local completion/broadcast reserve
 tpu_sigverify_workers = 0
 
 [consensus]

cmd/mithril/configcmd/configcmd_test.go

@@ -13,6 +13,10 @@ func TestStarterConfigSignatureVerification(t *testing.T) {
 		v := viper.New()
 		v.SetConfigType("toml")
 		require.NoError(t, v.ReadConfig(strings.NewReader(generateStarterConfig(validator))))
+		for _, key := range []string{"validator.wait_to_vote_slot", "validator.tpu_max_buffered_transactions", "validator.block_completion_reserve_ms"} {
+			require.True(t, v.IsSet(key), key)
+			require.Zero(t, v.GetInt(key), key)
+		}
 		require.Equal(t, "auto", v.GetString("sigverify.backend"))
 		require.True(t, v.IsSet("sigverify.workers"))
 		require.Zero(t, v.GetInt("sigverify.workers"))

cmd/mithril/node/node.go

@@ -126,6 +126,9 @@ var (
 	pprofPort      int64
 	blockstorePath string
 	txParallelism  int64
+	// streamingMaxOpenMs is --streaming-max-open-ms; resolved into
+	// replay.StreamingExecutionCfg.MaxOpenAge with the other [replay] keys.
+	streamingMaxOpenMs int
 
 	debugTxs                       []string
 	debugAcctWrites                []string
@@ -538,6 +541,14 @@ func init() {
 	// [replay] section flags
 	Run.Flags().Int64Var(&txParallelism, "txpar", 0, "Transaction execution workers (>0 enables topsort parallelism; explicit 0 is sequential; unset validator mode defaults to 2x CPU cores)")
 	Run.Flags().Int64Var(&numReplaySlots, "num-slots", 0, "Number of slots to replay (0 = run continuously)")
+	Run.Flags().BoolVar(&replay.StreamingExecutionCfg.Enabled, "streaming-execution", false,
+		"Execute Turbine blocks while their shreds arrive (Alpenglow validator/verifying modes only; the complete block remains authoritative and any mismatch falls back to whole-block execution)")
+	Run.Flags().IntVar(&replay.StreamingExecutionCfg.Workers, "streaming-workers", 0,
+		"Streaming execution workers per transaction group (0 = min(txpar, 4))")
+	Run.Flags().IntVar(&replay.StreamingExecutionCfg.MinGroupBatches, "streaming-min-group-batches", 0,
+		"Contiguous decoded batches to accumulate before a streaming group executes (0 or 1 = execute as batches arrive)")
+	Run.Flags().IntVar(&streamingMaxOpenMs, "streaming-max-open-ms", 0,
+		"Discard a streaming bank whose block has not completed after this many milliseconds (0 = 2000)")
 	Run.Flags().Int64VarP(&endSlot, "end-slot", "e", -1, "Block at which to stop replaying, inclusive (-1 = run continuously)")
 
 	// [consensus] section flags
@@ -1165,6 +1176,13 @@ func initConfigAndBindFlags(cmd *cobra.Command) error {
 	}
 	resolvedSigverifyBackend = resolved
 	sbpf.UsePool = getBool("use-pool", "tuning.use_pool")
+	// [tuning] streaming execution (off by default; Alpenglow turbine only).
+	replay.StreamingExecutionCfg.Enabled = getBool("streaming-execution", "tuning.streaming_execution")
+	replay.StreamingExecutionCfg.Workers = getInt("streaming-workers", "tuning.streaming_workers")
+	replay.StreamingExecutionCfg.MinGroupBatches = getInt("streaming-min-group-batches", "tuning.streaming_min_group_batches")
+	if ms := getInt("streaming-max-open-ms", "tuning.streaming_max_open_ms"); ms > 0 {
+		replay.StreamingExecutionCfg.MaxOpenAge = time.Duration(ms) * time.Millisecond
+	}
 	accountsdb.StoreAccountsWorkers = getInt("store-accounts-workers", "tuning.store_accounts_workers")
 	accountsdb.ProgramCacheMaxMB = getInt("program-cache-max-mb", "tuning.program_cache_max_mb")
 	if accountsdb.ProgramCacheMaxMB <= 0 {

config.example.toml

@@ -511,6 +511,21 @@ name = "mithril"
     # Zstd decoder concurrency (defaults to NumCPU)
     # zstd_decoder_concurrency = 16
 
+    # Streaming execution (Alpenglow, native turbine only): execute a block's
+    # entry batches while its remaining shreds arrive, on a speculative bank
+    # over the executed parent. The complete block stays authoritative — the
+    # executed prefix must be the block's own transactions (pointer identity)
+    # or the bank is discarded and the block executes whole. Off by default.
+    # streaming_execution = false
+    # Execution workers per streaming group (0 = min(txpar, 4)).
+    # streaming_workers = 0
+    # Contiguous decoded batches to accumulate before a group executes
+    # (0 or 1 = execute as each batch arrives).
+    # streaming_min_group_batches = 0
+    # Discard a speculative bank whose block has not completed after this
+    # many milliseconds (0 = 2000).
+    # streaming_max_open_ms = 0
+
     # Snapshot bootstrap I/O tuning.
     # These defaults deliberately avoid flooding a single NVMe with hundreds of
     # concurrent writes. Increase cautiously on very fast multi-disk systems.

docs/leader_block_packing.md

@@ -81,7 +81,14 @@ The corresponding flags are `--tpu-max-buffered-transactions` and
 `--leader-completion-reserve-ms`. The measured full-prefill trial used 262,144
 queue entries and a 60ms reserve. Those are opt-in tuning values; defaults stay
 unchanged. A larger queue uses additional memory for owned wire, decoded and
-prepared objects. A shorter reserve needs measured local finalization/broadcast
+prepared objects. `BenchmarkReadonlyPairPreparationMemory` measured 728 allocated
+bytes per preparation (9 allocations) on Go 1.26.4 arm64 for the 198-byte fixture.
+That is 91 MiB of allocation volume for 131,072 preparations, or 182 MiB for
+262,144, in addition to wire/decoded transactions and queue indexes. Allocation
+volume includes temporary preparation storage: it is not retained heap or RSS.
+More accounts/instructions increase the footprint; these are not worst-case caps.
+Reproduce with `go test ./pkg/blockprod -run '^$' -bench '^BenchmarkReadonlyPairPreparationMemory$' -benchmem`.
+A shorter reserve needs measured local finalization/broadcast
 margin and does not change the protocol deadline. Shifting completion also
 shifts later bank start times, so it does not add the same packing time to all
 four blocks.

docs/rewards-unwind-retirement.md

@@ -0,0 +1,58 @@
+# Retiring durable rewards bookkeeping
+
+A completed partitioned-rewards distribution used to leave its in-memory
+descriptor alive for the rest of the replay attempt. The fork-switch guard
+rejects any such descriptor because account-overlay unwind cannot restore the
+consumed spool or its distribution counters. This is necessary while completion
+is speculative, but unnecessarily forces checkpoint replay after completion
+has become durable.
+
+Replay now observes the inactive EpochRewards sysvar in a successfully executed
+bank's immutable snapshot, with zero partitions remaining. It remembers that
+bank's slot and the exact distribution descriptor. Only applying a successful
+durable fold through that slot retires the descriptor. Later bank observations
+do not move the completion slot forward. A new descriptor/epoch invalidates the
+old evidence; missing sysvars or unknown completion retain the old fallback.
+
+## Safety and recovery contract
+
+- Completion in memory, certificate finality, and submitting a fold do not
+  authorize retirement. Failed folds leave the durable watermark unchanged.
+- Active distribution and completed-but-not-durable distribution retain the
+  existing rewards guard. No spool reconstruction or rewards rollback is added.
+- After retirement, in-memory switches still require the existing epoch,
+  vote/stake-cache, parent-context, sysvar and transaction-status checks.
+  Switches at/below the durable watermark still require durable recovery.
+- Completion evidence is replay-thread-owned and process-local. It does not
+  change checkpoint formats, signing reservations, persisted vote history,
+  clean-shutdown rules or restart authorization. Restart retains the existing
+  persisted EpochRewards validation. No extra file or disk sync is introduced.
+
+## Incident motivating the change
+
+On Zen 5, distribution completed at slot 3,942,001. At a later parent-linked
+switch, the durable checkpoint was already 3,944,067; child 3,944,076 selected
+parent 3,944,073, abandoning the suffix from 3,944,074. The remaining descriptor
+forced the rewards-window fallback even though completion was below the root.
+Checkpoint recovery re-fetched previously received blocks, with logged waits
+of 2.739 seconds and 0.967 seconds. A buffered 665-transaction block waited
+3,613.510 ms for replay admission and then executed in 7.520 ms.
+
+These are incident observations, not a before/after benchmark or a measurement
+of checkpoint encoding/fsync time. Thirteen observed FAST aggregates omitted
+our vote during the recovery interval; that does not prove absence from every
+FAST aggregate or a single cause for all thirteen omissions. No live latency
+improvement is established until a comparable switch exercises the new path.
+
+## Validation
+
+`rewards_retirement_test.go` covers active/missing bank state, unknown completion,
+the exact durable boundary, later-bank observations, generation changes, failed
+and successful folds, and an exact-parent unwind after retirement (including
+account values, resume state and immutable rewards sysvars). Existing unwind
+tests still require fallback for zero-remaining bookkeeping without retirement,
+cross-epoch switches, dirty vote/stake caches and invalid parent snapshots.
+
+Full replay/rewards race suites passed locally and in the combined native
+build; native node recovery/checkpoint race tests, vet and validator build also
+passed. These are software tests, not mainnet power-loss qualification.

docs/status-checkpoint-capture.md

@@ -0,0 +1,98 @@
+# Transaction-status checkpoint capture and encoding
+
+Replay captures immutable lineage and coverage metadata before submitting a
+checkpoint to the promotion worker. Sorting, encoding and writing happen on
+that worker. Capture does not retain parent links outside the selected window.
+Publication and durable-root ordering are unchanged.
+
+Each node memoizes its canonical encoded body on first serialization. Capture
+and pruning share the same cache object when copying a node header; they never
+copy a used synchronization primitive. Encoding depends on the immutable slot,
+block-ID presence/value and status delta, not its parent link. Concurrent
+encoders synchronize through `sync.Once` without taking the live cache lock.
+Each snapshot still constructs its own coverage header and returns an owned
+output buffer. The MTS2 format and restore validation are unchanged.
+
+The cache retains roughly one extra encoded window (30 MB for 1.5 million
+keys), plus any nodes pinned by older views. There is no global encoding map:
+caches become collectible with their last node/view. A completely new window
+still pays for all sorting. Output copying and checkpoint I/O remain necessary.
+
+## Encoding benchmark
+
+`BenchmarkTransactionStatusCheckpointEncoding` uses a 300-root window with
+5,000 keys per root (1.5 million keys, roughly 30 MB encoded). Each iteration
+replaces the specified number of roots. Fixture creation and initial warming
+are excluded; new node headers, sorting and output allocations are included.
+The baseline is the original uncached wire encoder retained in tests.
+
+Apple M4 Pro, Go 1.26.4, one caller, GOMAXPROCS=12; medians of three runs:
+
+| New roots per checkpoint | Original encoding | Cached encoding |
+| --- | ---: | ---: |
+| 1 | 158.05 ms | 1.35 ms |
+| 8 | 157.14 ms | 5.09 ms |
+| 32 | 155.62 ms | 17.51 ms |
+| 128 (default fold cadence) | 153.74 ms | 67.11 ms |
+| 300 (entirely new) | 155.90 ms | 156.29 ms |
+
+At the default cadence, allocated bytes per encoding fell from 99.12 MB to
+57.33 MB; this excludes retained heap. These are encoding measurements, not
+end-to-end fold/replay timings or live FAST improvements. Data distribution
+matters: newly rooted large blocks can account for most keys in the window.
+
+Run `go test ./pkg/replay -run '^$' -bench '^BenchmarkTransactionStatusCheckpointEncoding$' -benchmem -benchtime=1s -count=3`.
+
+Tests compare exact bytes with the original encoder across coverage flags,
+block IDs and sorted groups; check concurrent encoding during pruning/unwind;
+verify cache sharing before and after warming; and restore checkpoints after
+callers mutate their own output buffers. The replay race suite and vet pass.
+
+Related behavior: [status expiry](transaction-status-expiry.md) and
+[status publication](transaction-status-publication.md).
+
+## Native Zen 5 validation
+
+AMD Ryzen 7 9700X, Go 1.26.4, GOMAXPROCS=2, Nice 15 and a two-core CPU quota,
+while the validator continued its normal workload. Same moving-window fixture;
+three samples per case, medians below. This compares the original uncached
+encoder with memoization, not the whole status-publication change against dev.
+
+| New roots per checkpoint | Original encoding | Cached encoding |
+| --- | ---: | ---: |
+| 1 | 195.34 ms | 3.73 ms |
+| 8 | 195.15 ms | 8.56 ms |
+| 32 | 194.77 ms | 23.54 ms |
+| 128 (default fold cadence) | 194.52 ms | 85.02 ms |
+| 300 (entirely new) | 201.88 ms | 198.55 ms |
+
+The default-cadence result is approximately 2.3x, with the same 99.12 → 57.33 MB
+allocation reduction. Cold/all-new windows remain roughly unchanged. Native
+combined race suites, vet and the validator build passed. These are historical staging measurements. They do not establish an isolated
+live reduction in durable-root lag or missed FAST votes.
+
+## Fold admission before collecting account writes
+
+Replay checks for a checkpoint batch on every iteration, including skipped
+slots. `WorkingSet.PromotionChunk` first counts eligible held slots under its
+read lock. If fewer than the configured batch size are available, ordinary
+admission returns nil without allocating account-pointer lists. When ready,
+it collects only the oldest batch, not the entire eligible suffix. Forced
+partial folds still collect the available prefix.
+
+This preflight is not a finality shortcut or a new recovery policy. Replay's
+existing finality/verification gates supply the upper bound. Selection and
+collection hold the same lock; account pointers retain their existing ownership
+contract. Preparation does not prune the suffix or advance the durable root.
+The worker's write/commit order, required resume context, checkpoint reference
+validation, completion bookkeeping, and forced shutdown/epoch-boundary paths
+are unchanged.
+
+`BenchmarkBuildFoldJobWaitingForBatch` holds 127 slots with 512 account writes
+each while waiting for the default 128-slot batch. On Ryzen 9700X,
+GOMAXPROCS=8, three 300 ms runs, median admission-check time fell from 426 µs
+to 31.8 ns; 627,008 bytes and 134 allocations per rejected preparation became
+zero. This measures an ineligible batch check, not encoding, disk I/O, or a
+ready checkpoint. Boundary tests cover gaps, the finality upper bound, a full
+batch, forced partial batches, and selection after promotion; existing replay
+checkpoint/recovery tests cover the unchanged durable path.

docs/status-checkpoint-expiry-evidence.md

@@ -0,0 +1,19 @@
+# Status Checkpoint Expiry: benchmark evidence
+
+The maintained subsystem documentation and reusable Go benchmarks describe the
+implementation and reproduction method. Historical raw results and session
+notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/a511ad3b0bc77cf8b5ae4ee16359ac6b453fc7bf)
+(tag `review-evidence-20260916-status-checkpoint-expiry`). They are omitted from this proposed merge.
+
+[Historical result files](https://github.com/Overclock-Validator/mithril/tree/a511ad3b0bc77cf8b5ae4ee16359ac6b453fc7bf/docs/results)
+
+Measurements retain their original baselines. Rebasing onto PR #278 does not
+turn an intermediate-version benchmark into a comparison with the new base.
+Component timings and short live observations do not establish sustained FAST
+inclusion gains. The final review description records validation of the rebased
+source separately from historical benchmark results.
+
+The tagged snapshot also preserves the later 64-partition visible-status-map
+experiment. That experiment is deliberately excluded from this review: it added
+preparation work and did not demonstrate an overall large-block p99 benefit.
+Earlier publication preparation and immutable-node encoding reuse remain.