alpenglow: improve certificate processing, delivery and signing persistence

.github/workflows/go_build.yml

@@ -17,3 +17,31 @@ jobs:
 
       - name: Build
         run: go build -v ./cmd/mithril
+
+  regression-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    env:
+      GOMAXPROCS: "2"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.26.4
+
+      - name: Voting, checkpoint, streaming and scheduler race regressions
+        # Run the complete affected package suites, including subprocess crash
+        # recovery and cancellation tests. The independent sealevel suite has
+        # known base-branch failures documented in the validation report.
+        run: >-
+          go test -race -p 2 -count=1
+          ./pkg/alpenglow ./pkg/consensus ./pkg/replay
+          ./pkg/turbine ./pkg/sigverify ./pkg/blockprod/...
+          ./cmd/mithril/node ./cmd/mithril/configcmd
+
+      - name: Vote-program deque ownership race regression
+        run: go test -race -count=1 ./pkg/sealevel -run '^TestProcessNewVoteStateOwnsRetainedDeque$'

cmd/mithril/node/node.go

@@ -103,6 +103,9 @@ var (
 	validatorTPUQUICBind            string
 	validatorAdvertisedIP           string
 	validatorSigverifyWorkers       int
+	validatorWaitToVoteSlot         uint64
+	validatorReservedHistory        bool
+	validatorInitializeReservation  bool
 
 	// Mode thresholds
 	blockNearTipThreshold        int // Enter near-tip when gap <= this
@@ -547,6 +550,9 @@ func init() {
 	Run.Flags().StringVar(&validatorTPUQUICBind, "tpu-quic-bind-addr", "", "Validator TPU QUIC listen address (default 0.0.0.0:8004)")
 	Run.Flags().StringVar(&validatorAdvertisedIP, "validator-advertised-ip", "", "Public IP advertised for validator TPU QUIC")
 	Run.Flags().IntVar(&validatorSigverifyWorkers, "tpu-sigverify-workers", 0, "TPU signature verification workers (0 = GOMAXPROCS)")
+	Run.Flags().BoolVar(&validatorReservedHistory, "reserved-vote-history", false, "Use durable signing reservations with unsynchronized per-vote history writes")
+	Run.Flags().BoolVar(&validatorInitializeReservation, "initialize-vote-reservation", false, "Enroll complete synchronous vote history in reserved mode (one-time migration)")
+	Run.Flags().Uint64Var(&validatorWaitToVoteSlot, "wait-to-vote-slot", 0, "Do not cast new votes below this slot; the automatic startup cutoff still applies (0 = automatic only)")
 
 	// [tuning] section flags
 	Run.Flags().Uint64Var(&paramArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)")
@@ -639,6 +645,11 @@ func initConfigAndBindFlags(cmd *cobra.Command) error {
 	if err := config.InitConfig(); err != nil {
 		return err
 	}
+	if slot, err := configuredWaitToVoteSlot(cmd); err != nil {
+		return err
+	} else {
+		validatorWaitToVoteSlot = slot
+	}
 
 	// Check if a CLI flag was explicitly set by the user
 	flagChanged := func(name string) bool {
@@ -2642,52 +2653,9 @@ postBootstrap:
 		}
 		global.SeedWallClockSlot(wallClockSeed)
 		startupWallSlot := global.WallClockSlot()
-		waitToVoteSlot := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots
-		if waitToVoteSlot <= math.MaxUint64-2*alpenglow.LeaderWindowSlots {
-			waitToVoteSlot += 2 * alpenglow.LeaderWindowSlots
-		} else {
-			waitToVoteSlot = math.MaxUint64
-		}
-		mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d wait_to_vote=%d", startupWallSlot, waitToVoteSlot)
+		waitToVoteSlot := effectiveWaitToVoteSlot(startupWallSlot, validatorWaitToVoteSlot)
+		mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d configured_wait_to_vote=%d wait_to_vote=%d", startupWallSlot, validatorWaitToVoteSlot, waitToVoteSlot)
 
-		identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey()
-		if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{
-			Identity:        validatorIdentity,
-			AuthorizedVoter: validatorAuthorizedVoter,
-			VoteAccount:     validatorVoteAccount,
-			HistoryDir:      blockstorePath,
-			EpochForSlot:    epochSchedule.GetEpoch,
-			SlotDuration:    blockprod.AlpenglowSlotDuration,
-			WaitToVoteSlot:  waitToVoteSlot,
-			ReadyToVote: func(slot uint64) bool {
-				wallSlot := global.WallClockSlot()
-				if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok {
-					wallSlot = liveSlot
-				}
-				return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots
-			},
-			Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer {
-				peers := make([]alpenglow.VotorPeer, 0, len(validators))
-				seen := make(map[solana.PublicKey]struct{}, len(validators))
-				for _, validator := range validators {
-					if validator.Stake == 0 || validator.NodePubkey == identityPubkey {
-						continue
-					}
-					addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey)
-					if !ok {
-						continue
-					}
-					if _, duplicate := seen[validator.NodePubkey]; duplicate {
-						continue
-					}
-					seen[validator.NodePubkey] = struct{}{}
-					peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr})
-				}
-				return peers
-			},
-		}); err != nil {
-			klog.Fatalf("enable Alpenglow voting: %v", err)
-		}
 		broadcaster, err := turbine.NewTurbineBroadcaster(turbine.TurbineBroadcasterConfig{
 			Self:          solana.PrivateKey(validatorIdentity).PublicKey(),
 			Peers:         sharedGossip,
@@ -2744,6 +2712,50 @@ postBootstrap:
 			mlog.Log.Warnf("validator gossip TPU advertisement: %v", err)
 		}
 
+		// Bind and validate local transports before consuming the durable clean
+		// voting marker. Startup configuration failures must not force recovery.
+		identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey()
+		if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{
+			Identity:                  validatorIdentity,
+			AuthorizedVoter:           validatorAuthorizedVoter,
+			VoteAccount:               validatorVoteAccount,
+			HistoryDir:                blockstorePath,
+			ReservedHistory:           validatorReservedHistory,
+			InitializeVoteReservation: validatorInitializeReservation,
+			Genesis:                   solana.MustHashFromBase58(networkGenesisHash),
+			EpochForSlot:              epochSchedule.GetEpoch,
+			SlotDuration:              blockprod.AlpenglowSlotDuration,
+			WaitToVoteSlot:            waitToVoteSlot,
+			ReadyToVote: func(slot uint64) bool {
+				wallSlot := global.WallClockSlot()
+				if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok {
+					wallSlot = liveSlot
+				}
+				return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots
+			},
+			Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer {
+				peers := make([]alpenglow.VotorPeer, 0, len(validators))
+				seen := make(map[solana.PublicKey]struct{}, len(validators))
+				for _, validator := range validators {
+					if validator.Stake == 0 || validator.NodePubkey == identityPubkey {
+						continue
+					}
+					addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey)
+					if !ok {
+						continue
+					}
+					if _, duplicate := seen[validator.NodePubkey]; duplicate {
+						continue
+					}
+					seen[validator.NodePubkey] = struct{}{}
+					peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr})
+				}
+				return peers
+			},
+		}); err != nil {
+			klog.Fatalf("enable Alpenglow voting: %v", err)
+		}
+
 		rewardBuilder := rewardcerts.NewBuilder(rewardcerts.BuilderConfig{
 			RootSlot:    global.Slot,
 			BeforeBuild: consensusEngine.FlushAlpenglowRewardVotes,
@@ -2796,6 +2808,7 @@ postBootstrap:
 				}
 			},
 			ProductionParent: consensusEngine.AlpenglowBlockProductionParent,
+			CanSignSlot:      consensusEngine.AlpenglowCanSignLeaderSlot,
 			CurrentSlot: func() uint64 {
 				if slot, ok := consensusEngine.AlpenglowLiveSlot(); ok {
 					return slot

cmd/mithril/node/vote_startup.go

@@ -0,0 +1,40 @@
+package node
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+
+	"github.com/Overclock-Validator/mithril/pkg/alpenglow"
+	"github.com/Overclock-Validator/mithril/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+func configuredWaitToVoteSlot(cmd *cobra.Command) (uint64, error) {
+	if flag := cmd.Flags().Lookup("wait-to-vote-slot"); flag != nil && flag.Changed {
+		return cmd.Flags().GetUint64("wait-to-vote-slot")
+	}
+	const key = "validator.wait_to_vote_slot"
+	if !config.IsSet(key) {
+		return 0, nil
+	}
+	// Unlike GetUint64, parsing explicitly must not turn an invalid operator
+	// cutoff into zero and silently remove the requested voting restriction.
+	slot, err := strconv.ParseUint(config.GetString(key), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("%s must be an unsigned 64-bit slot: %w", key, err)
+	}
+	return slot, nil
+}
+
+// The operator cutoff can postpone voting but cannot weaken the existing
+// startup guard. Equality permits voting, subject to all other Votor checks.
+func effectiveWaitToVoteSlot(startupWallSlot, configured uint64) uint64 {
+	automatic := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots
+	if automatic <= math.MaxUint64-2*alpenglow.LeaderWindowSlots {
+		automatic += 2 * alpenglow.LeaderWindowSlots
+	} else {
+		automatic = math.MaxUint64
+	}
+	return max(automatic, configured)
+}

cmd/mithril/node/vote_startup_test.go

@@ -0,0 +1,73 @@
+package node
+
+import (
+	"math"
+	"strings"
+	"testing"
+
+	"github.com/Overclock-Validator/mithril/pkg/config"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfiguredWaitToVoteSlot(t *testing.T) {
+	for _, tc := range []struct {
+		name, toml, cli string
+		want            uint64
+		invalid         bool
+	}{
+		{name: "default"},
+		{name: "toml", toml: "wait_to_vote_slot = 1234", want: 1234},
+		{name: "cli wins", toml: "wait_to_vote_slot = 1234", cli: "5678", want: 5678},
+		{name: "explicit zero wins", toml: "wait_to_vote_slot = 1234", cli: "0"},
+		{name: "maximum CLI", cli: "18446744073709551615", want: math.MaxUint64},
+		{name: "negative TOML", toml: "wait_to_vote_slot = -1", invalid: true},
+		{name: "fractional TOML", toml: "wait_to_vote_slot = 1.5", invalid: true},
+		{name: "malformed TOML value", toml: `wait_to_vote_slot = "oops"`, invalid: true},
+		{name: "empty TOML value", toml: `wait_to_vote_slot = ""`, invalid: true},
+		{name: "overflow TOML value", toml: `wait_to_vote_slot = "18446744073709551616"`, invalid: true},
+		{name: "negative CLI", cli: "-1", invalid: true},
+		{name: "overflow CLI", cli: "18446744073709551616", invalid: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			viper.Reset()
+			t.Cleanup(viper.Reset)
+			config.ApplyDefaults(viper.GetViper())
+			viper.SetConfigType("toml")
+			require.NoError(t, viper.ReadConfig(strings.NewReader("[validator]\n"+tc.toml)))
+			cmd := &cobra.Command{}
+			cmd.Flags().Uint64("wait-to-vote-slot", 0, "")
+			var err error
+			if tc.cli != "" {
+				err = cmd.Flags().Set("wait-to-vote-slot", tc.cli)
+			}
+			var got uint64
+			if err == nil {
+				got, err = configuredWaitToVoteSlot(cmd)
+			}
+			if tc.invalid {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tc.want, got)
+		})
+	}
+	require.NotNil(t, Run.Flags().Lookup("wait-to-vote-slot"))
+}
+
+func TestEffectiveWaitToVoteSlot(t *testing.T) {
+	for _, tc := range []struct{ startup, configured, want uint64 }{
+		{100, 0, 108},
+		{103, 0, 108},
+		{103, 104, 108},
+		{103, 108, 108},
+		{103, 123, 123}, // Operator cutoff need not align with a leader window.
+		{103, math.MaxUint64, math.MaxUint64},
+		{math.MaxUint64 - 7, 0, math.MaxUint64},
+		{math.MaxUint64, 0, math.MaxUint64},
+	} {
+		require.Equal(t, tc.want, effectiveWaitToVoteSlot(tc.startup, tc.configured), "%+v", tc)
+	}
+}

config.example.toml

@@ -328,6 +328,16 @@ name = "mithril"
     # Signature-verification workers (0 = GOMAXPROCS).
     tpu_sigverify_workers = 0
 
+    # Optional minimum slot for NEW votes (inclusive), also --wait-to-vote-slot.
+    # Useful when rejoining after recovery. CLI overrides this setting.
+    # Zero adds no operator cutoff; the automatic startup cutoff and normal
+    # consensus checks still apply. A lower value cannot bypass those checks.
+    # Replay/repair continue while waiting. Previously recorded, authenticated
+    # votes can still be restored/rebroadcast under the existing recovery rules.
+    # This does not coordinate a cluster restart or wait for supermajority,
+    # and does not allow resetting a corrupt vote-history file.
+    wait_to_vote_slot = 0
+
 # ============================================================================
 # [consensus] - Alpenglow Consensus
 # ============================================================================

docs/alpenglow_branch_engine.md

@@ -160,6 +160,31 @@ mixed (heterogeneous-client) or Mithril-only cluster identically:
   timeouts, vote signing/transmission, durable vote-history persistence, and
   standstill participation.
 
+## Replay-observer diagnostics
+
+The observer retains certificate history for deduplication and match/mismatch
+reporting. A separate bounded index contains only retained, block-bearing
+certificates that have not yet been reconciled against replay. Reconciliation
+removes an entry after either a match or mismatch; eviction removes it together
+with the historical certificate. Hashless/skipped replay cannot reconcile a
+block-bearing certificate. Pending counts and age/window statistics retain the
+same semantics, but scan unresolved entries rather than completed history.
+
+This index is disposable, process-local diagnostic state. It neither authorizes
+votes nor substitutes for verified certificates, the chain tracker's finality
+checks, durable signing bounds, vote history, or checkpoint recovery. Those
+checks and persistence contracts are unchanged.
+
+`BenchmarkObserverEmptyReplay` measures observer work for an empty block, with
+or without four preceding skipped slots, against 4,096 retained certificates.
+It covers 0, 32, and 4,096 unresolved entries. On Ryzen 9700X (GOMAXPROCS=8,
+three 300 ms runs), median time for the four-skips-plus-empty case with 32
+unresolved entries was 686.4 µs before the index and 1.87 µs afterward. With
+all 4,096 entries unresolved it was 380.5 → 159.3 µs. These are component
+benchmarks; they exclude execution, certificate cryptography, network delivery,
+and end-to-end FAST inclusion. Live comparisons must account for observer
+history warming after a restart and different leader/skip patterns.
+
 ## What this proves — and does not
 
 The certificate layer proves which block *data* the cluster settled on. In

docs/certificate-processing-evidence.md

@@ -0,0 +1,14 @@
+# Certificate Processing: benchmark evidence
+
+The maintained subsystem documentation and reusable Go benchmarks describe the
+implementation and reproduction method. Historical raw results and session
+notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc)
+(tag `review-evidence-20260916-certificate-processing`). They are omitted from this proposed merge.
+
+[Historical result files](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc/docs/results)
+
+Measurements retain their original baselines. Rebasing onto PR #278 does not
+turn an intermediate-version benchmark into a comparison with the new base.
+Component timings and short live observations do not establish sustained FAST
+inclusion gains. The final review description records validation of the rebased
+source separately from historical benchmark results.