rewards: preserve inactive stake credits and checkpoint recovery

.github/workflows/go_build.yml

@@ -17,3 +17,6 @@ jobs:
 
       - name: Build
         run: go build -v ./cmd/mithril
+
+      - name: Rewards and replay regressions
+        run: GOMAXPROCS=2 go test -race -p 2 -count=1 ./pkg/rewards ./pkg/replay

docs/rewards-unwind-retirement.md

@@ -0,0 +1,58 @@
+# Retiring durable rewards bookkeeping
+
+A completed partitioned-rewards distribution used to leave its in-memory
+descriptor alive for the rest of the replay attempt. The fork-switch guard
+rejects any such descriptor because account-overlay unwind cannot restore the
+consumed spool or its distribution counters. This is necessary while completion
+is speculative, but unnecessarily forces checkpoint replay after completion
+has become durable.
+
+Replay now observes the inactive EpochRewards sysvar in a successfully executed
+bank's immutable snapshot, with zero partitions remaining. It remembers that
+bank's slot and the exact distribution descriptor. Only applying a successful
+durable fold through that slot retires the descriptor. Later bank observations
+do not move the completion slot forward. A new descriptor/epoch invalidates the
+old evidence; missing sysvars or unknown completion retain the old fallback.
+
+## Safety and recovery contract
+
+- Completion in memory, certificate finality, and submitting a fold do not
+  authorize retirement. Failed folds leave the durable watermark unchanged.
+- Active distribution and completed-but-not-durable distribution retain the
+  existing rewards guard. No spool reconstruction or rewards rollback is added.
+- After retirement, in-memory switches still require the existing epoch,
+  vote/stake-cache, parent-context, sysvar and transaction-status checks.
+  Switches at/below the durable watermark still require durable recovery.
+- Completion evidence is replay-thread-owned and process-local. It does not
+  change checkpoint formats, signing reservations, persisted vote history,
+  clean-shutdown rules or restart authorization. Restart retains the existing
+  persisted EpochRewards validation. No extra file or disk sync is introduced.
+
+## Incident motivating the change
+
+On Zen 5, distribution completed at slot 3,942,001. At a later parent-linked
+switch, the durable checkpoint was already 3,944,067; child 3,944,076 selected
+parent 3,944,073, abandoning the suffix from 3,944,074. The remaining descriptor
+forced the rewards-window fallback even though completion was below the root.
+Checkpoint recovery re-fetched previously received blocks, with logged waits
+of 2.739 seconds and 0.967 seconds. A buffered 665-transaction block waited
+3,613.510 ms for replay admission and then executed in 7.520 ms.
+
+These are incident observations, not a before/after benchmark or a measurement
+of checkpoint encoding/fsync time. Thirteen observed FAST aggregates omitted
+our vote during the recovery interval; that does not prove absence from every
+FAST aggregate or a single cause for all thirteen omissions. No live latency
+improvement is established until a comparable switch exercises the new path.
+
+## Validation
+
+`rewards_retirement_test.go` covers active/missing bank state, unknown completion,
+the exact durable boundary, later-bank observations, generation changes, failed
+and successful folds, and an exact-parent unwind after retirement (including
+account values, resume state and immutable rewards sysvars). Existing unwind
+tests still require fallback for zero-remaining bookkeeping without retirement,
+cross-epoch switches, dirty vote/stake caches and invalid parent snapshots.
+
+Full replay/rewards race suites passed locally and in the combined native
+build; native node recovery/checkpoint race tests, vet and validator build also
+passed. These are software tests, not mainnet power-loss qualification.

pkg/replay/block.go

@@ -1747,6 +1747,7 @@ func ReplayBlocks(
 	var unwoundParentBankSysvars *sealevel.BankSysvars
 	var partitionedEpochRewardsEnabled bool
 	var partitionedRewardsInfo *rewards.PartitionedRewardDistributionInfo
+	var rewardsCompletion partitionedRewardsCompletion
 	var featuresActivatedInFirstSlot []*accounts.Account
 	var parentFeaturesActivatedInFirstSlot []*accounts.Account
 
@@ -1928,8 +1929,8 @@ func ReplayBlocks(
 	var highestExecutedSlot uint64 // highest slot ProcessBlock has executed; bounds the promotion-gate walk
 	// While partitioned rewards distribute, promotion holds below the boundary
 	// block so a crash-resume always re-runs it (the distribution bookkeeping is
-	// RAM-only and not reconstructible mid-window). Self-clears when the window
-	// completes (NumRewardPartitionsRemaining reaches 0).
+	// RAM-only and not reconstructible mid-window). Release requires a verified
+	// completion bank, committed atomically with the whole rewards window.
 	var rewardsHoldBelowSlot uint64
 	// Alpenglow finality identities captured at observe/ingest time for the promotion
 	// gate (the tracker's own state may be pruned by promotion time). Pruned as slots
@@ -2063,6 +2064,10 @@ func ReplayBlocks(
 		mithrilState.LastRootedSlot = promotedThrough
 		mithrilState.LastRootedBankhash = rootedCtx.Bankhash
 		mithrilState.LastRootedContext = rootedCtx
+		if rewardsCompletion.retire(&partitionedRewardsInfo, promotedThrough) {
+			rewardsHoldBelowSlot = 0
+			mlog.Log.Infof("epoch rewards bookkeeping retired through durable slot %d; later fork switches may unwind in memory", promotedThrough)
+		}
 		if transactionStatuses.Root(promotedThrough) {
 			mlog.Log.Infof("transaction status cache reconstructed complete %d-root coverage through durable slot %d",
 				maxTransactionStatusRoots, promotedThrough)
@@ -2170,13 +2175,9 @@ func ReplayBlocks(
 		}
 		promoteThrough := safePromoteTarget(lastRootedWatermark, verifierRequired, verifiedWM, replayDivergenceFloor)
 		// Partitioned-rewards window: hold promotion below the boundary block
-		// until every partition distributes, so a crash-resume re-runs the
-		// boundary and rebuilds the RAM-only distribution bookkeeping.
-		if rewardsHoldBelowSlot > 0 && partitionedRewardsInfo != nil && partitionedRewardsInfo.NumRewardPartitionsRemaining > 0 {
-			if promoteThrough >= rewardsHoldBelowSlot {
-				promoteThrough = rewardsHoldBelowSlot - 1
-			}
-		}
+		// until the completion bank verifies and is eligible to fold, so a
+		// failed distribution re-runs the boundary and rebuilds its bookkeeping.
+		promoteThrough = rewardsCompletion.limitPromotion(partitionedRewardsInfo, rewardsHoldBelowSlot, promoteThrough)
 		if promoteThrough <= mithrilState.LastRootedSlot {
 			// Operator signal: promotion is fully stalled (verifier lag,
 			// divergence floor, or rewards hold) while finality has run at
@@ -2207,6 +2208,8 @@ func ReplayBlocks(
 			mlog.Log.FileOnlyf("alpenglow gate: checked=%d matched=%d no_finality=%d no_local_id=%d",
 				gateStats.checked, gateStats.matched, gateStats.noFinality, gateStats.noLocalID)
 		}
+		// The finality gate can stop before the verified completion bank.
+		promoteThrough = rewardsCompletion.limitPromotion(partitionedRewardsInfo, rewardsHoldBelowSlot, promoteThrough)
 		if promoteThrough <= mithrilState.LastRootedSlot {
 			return false
 		}
@@ -2220,6 +2223,18 @@ func ReplayBlocks(
 			if res := promoter.drain(); res != nil {
 				applyFoldOutcome(res)
 			}
+			if rewardsHoldBelowSlot > 0 && partitionedRewardsInfo != nil && promoteThrough >= rewardsHoldBelowSlot {
+				job, jerr := unrootedTailState.buildRewardsCompletionFoldJob(rewardsCompletion.slot)
+				if jerr != nil {
+					mlog.Log.Errorf("rooted-durable: rewards completion fold: %v", jerr)
+					return false
+				}
+				if err := runFoldJob(unrootedTailState.committer, job); err != nil {
+					mlog.Log.Errorf("rooted-durable: rewards completion fold: %v", err)
+					return false
+				}
+				applyFoldOutcome(&foldResult{job: job})
+			}
 			promotedThrough, rootedCtx, perr := unrootedTailState.flush(promoteThrough)
 			if perr != nil {
 				mlog.Log.Errorf("rooted-durable: forced fold stopped at slot %d: %v", promotedThrough, perr)
@@ -2233,7 +2248,13 @@ func ReplayBlocks(
 		// when idle; completions are applied at the top of this function on a
 		// later iteration.
 		if !promoter.inFlight {
-			job, jerr := unrootedTailState.buildFoldJob(promoteThrough, false)
+			var job *foldJob
+			var jerr error
+			if rewardsHoldBelowSlot > 0 && partitionedRewardsInfo != nil && promoteThrough >= rewardsHoldBelowSlot {
+				job, jerr = unrootedTailState.buildRewardsCompletionFoldJob(rewardsCompletion.slot)
+			} else {
+				job, jerr = unrootedTailState.buildFoldJob(promoteThrough, false)
+			}
 			if jerr != nil {
 				mlog.Log.Errorf("rooted-durable: %v; watermark held back", jerr)
 				return false
@@ -2914,6 +2935,7 @@ func ReplayBlocks(
 				boundaryParentCtx = epochBoundaryParentCtx(acctsDb, block, currentEpoch, replayCtx.CurrentFeatures)
 			}
 			partitionedRewardsInfo = handleEpochTransition(acctsDb, partitionedEpochRewardsEnabled, boundaryParentCtx, replayCtx, epochSchedule, replayCtx.CurrentFeatures, block, currentEpoch, rpcc, dbgOpts)
+			rewardsCompletion = partitionedRewardsCompletion{}
 			currentEpoch = block.Epoch
 			justCrossedEpochBoundary = true
 			// While partitioned rewards are distributing, hold durable promotion
@@ -3026,6 +3048,7 @@ func ReplayBlocks(
 		}
 		// The successful child now owns its derived snapshot. Any later bank uses
 		// lastSlotCtx; the one-shot retained unwind bridge is no longer needed.
+		rewardsCompletion.observeBank(partitionedRewardsInfo, lastSlotCtx.BankSysvars())
 		unwoundParentBankSysvars = nil
 		postProcessBlockStart := processBlockEnd
 		statusViewStart := time.Now()

pkg/replay/promotion.go

@@ -388,6 +388,23 @@ type foldResult struct {
 	err error
 }
 
+// buildRewardsCompletionFoldJob puts the entire rewards window in one commit.
+// A normal batch cutoff inside that window would leave an active EpochRewards
+// checkpoint without the RAM-only spool bookkeeping needed to resume it. The
+// retained tail already bounds the size of this once-per-epoch fold.
+func (t *unrootedTail) buildRewardsCompletionFoldJob(through uint64) (*foldJob, error) {
+	whole := *t
+	whole.batchSlots = t.overlay.HeldSlots()
+	job, err := whole.buildFoldJob(through, true)
+	if err != nil {
+		return nil, err
+	}
+	if job == nil || job.through != through {
+		return nil, fmt.Errorf("rewards completion bank %d is absent from retained fold prefix", through)
+	}
+	return job, nil
+}
+
 // buildFoldJob snapshots the FIRST fold chunk of the rooted prefix <= through
 // (loop thread). force also takes a trailing partial chunk. Returns nil when
 // no chunk is ready. A missing chunk-top context is an error — a context-less

pkg/replay/rewards_retirement.go

@@ -0,0 +1,62 @@
+package replay
+
+import (
+	"github.com/Overclock-Validator/mithril/pkg/rewards"
+	"github.com/Overclock-Validator/mithril/pkg/sealevel"
+)
+
+// partitionedRewardsCompletion is replay-thread-owned, process-local evidence
+// that a successfully executed bank contains all effects of this distribution.
+// It is not a checkpoint or signing authority. Until that bank is durable,
+// tryInLoopUnwind must still reject even a zero-remaining distribution: its
+// spool has been consumed and cannot be rolled back with the account overlay.
+type partitionedRewardsCompletion struct {
+	info *rewards.PartitionedRewardDistributionInfo
+	slot uint64
+}
+
+// limitPromotion keeps the boundary replayable until a successfully verified
+// completion bank is eligible for promotion. Consuming the last spool changes
+// the RAM counter before footer verification and is not completion evidence.
+func (c *partitionedRewardsCompletion) limitPromotion(info *rewards.PartitionedRewardDistributionInfo, boundary, through uint64) uint64 {
+	if boundary == 0 || info == nil {
+		return through
+	}
+	if info.NumRewardPartitionsRemaining != 0 || c.info != info || c.slot == 0 || through < c.slot {
+		return min(through, boundary-1)
+	}
+	return through
+}
+
+// observeBank must run only after successful block execution/publication, using
+// that bank's immutable sysvars (never the speculative global sysvar cache).
+// If the first completed bank lacks evidence, recording a later descendant is
+// conservative: retirement then waits for that later bank to become durable.
+func (c *partitionedRewardsCompletion) observeBank(info *rewards.PartitionedRewardDistributionInfo, bank *sealevel.BankSysvars) {
+	if c.info != info {
+		*c = partitionedRewardsCompletion{info: info}
+	}
+	if info == nil || c.slot != 0 || info.NumRewardPartitionsRemaining != 0 || bank == nil || bank.Slot() == 0 {
+		return
+	}
+	epochRewards, ok := bank.EpochRewards()
+	if ok && !epochRewards.Active {
+		c.slot = bank.Slot()
+	}
+}
+
+// retire is called only when replay applies a successfully committed fold and
+// advances LastRootedSlot. Finality, an enqueued/in-flight fold, and a failed
+// commit do not acknowledge durability. At this boundary every rewards effect
+// is in AccountsDB; in-memory switches above it cannot undo distribution.
+// Switches at/below it still take durable recovery, whose persisted
+// EpochRewards validation remains unchanged. Restart loses this optional
+// evidence and reconstructs state through the existing recovery path.
+func (c *partitionedRewardsCompletion) retire(info **rewards.PartitionedRewardDistributionInfo, durableSlot uint64) bool {
+	if *info == nil || *info != c.info || c.slot == 0 || durableSlot < c.slot || (*info).NumRewardPartitionsRemaining != 0 {
+		return false
+	}
+	*info = nil
+	*c = partitionedRewardsCompletion{}
+	return true
+}