alpenglow: add opt-in signing reservations and ordered history persistence

.github/workflows/go_build.yml

@@ -17,3 +17,31 @@ jobs:
 
       - name: Build
         run: go build -v ./cmd/mithril
+
+  regression-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    permissions:
+      contents: read
+    env:
+      GOMAXPROCS: "2"
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: 1.26.4
+
+      - name: Voting, checkpoint, streaming and scheduler race regressions
+        # Run the complete affected package suites, including subprocess crash
+        # recovery and cancellation tests. The independent sealevel suite has
+        # known base-branch failures documented in the validation report.
+        run: >-
+          go test -race -p 2 -count=1
+          ./pkg/alpenglow ./pkg/consensus ./pkg/replay
+          ./pkg/turbine ./pkg/sigverify ./pkg/blockprod/...
+          ./cmd/mithril/node ./cmd/mithril/configcmd
+
+      - name: Vote-program deque ownership race regression
+        run: go test -race -count=1 ./pkg/sealevel -run '^TestProcessNewVoteStateOwnsRetainedDeque$'

cmd/mithril/node/node.go

@@ -103,6 +103,9 @@ var (
 	validatorTPUQUICBind            string
 	validatorAdvertisedIP           string
 	validatorSigverifyWorkers       int
+	validatorWaitToVoteSlot         uint64
+	validatorReservedHistory        bool
+	validatorInitializeReservation  bool
 
 	// Mode thresholds
 	blockNearTipThreshold        int // Enter near-tip when gap <= this
@@ -547,6 +550,9 @@ func init() {
 	Run.Flags().StringVar(&validatorTPUQUICBind, "tpu-quic-bind-addr", "", "Validator TPU QUIC listen address (default 0.0.0.0:8004)")
 	Run.Flags().StringVar(&validatorAdvertisedIP, "validator-advertised-ip", "", "Public IP advertised for validator TPU QUIC")
 	Run.Flags().IntVar(&validatorSigverifyWorkers, "tpu-sigverify-workers", 0, "TPU signature verification workers (0 = GOMAXPROCS)")
+	Run.Flags().BoolVar(&validatorReservedHistory, "reserved-vote-history", false, "Use durable signing reservations with unsynchronized per-vote history writes")
+	Run.Flags().BoolVar(&validatorInitializeReservation, "initialize-vote-reservation", false, "Enroll complete synchronous vote history in reserved mode (one-time migration)")
+	Run.Flags().Uint64Var(&validatorWaitToVoteSlot, "wait-to-vote-slot", 0, "Do not cast new votes below this slot; the automatic startup cutoff still applies (0 = automatic only)")
 
 	// [tuning] section flags
 	Run.Flags().Uint64Var(&paramArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)")
@@ -639,6 +645,11 @@ func initConfigAndBindFlags(cmd *cobra.Command) error {
 	if err := config.InitConfig(); err != nil {
 		return err
 	}
+	if slot, err := configuredWaitToVoteSlot(cmd); err != nil {
+		return err
+	} else {
+		validatorWaitToVoteSlot = slot
+	}
 
 	// Check if a CLI flag was explicitly set by the user
 	flagChanged := func(name string) bool {
@@ -2642,52 +2653,9 @@ postBootstrap:
 		}
 		global.SeedWallClockSlot(wallClockSeed)
 		startupWallSlot := global.WallClockSlot()
-		waitToVoteSlot := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots
-		if waitToVoteSlot <= math.MaxUint64-2*alpenglow.LeaderWindowSlots {
-			waitToVoteSlot += 2 * alpenglow.LeaderWindowSlots
-		} else {
-			waitToVoteSlot = math.MaxUint64
-		}
-		mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d wait_to_vote=%d", startupWallSlot, waitToVoteSlot)
+		waitToVoteSlot := effectiveWaitToVoteSlot(startupWallSlot, validatorWaitToVoteSlot)
+		mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d configured_wait_to_vote=%d wait_to_vote=%d", startupWallSlot, validatorWaitToVoteSlot, waitToVoteSlot)
 
-		identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey()
-		if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{
-			Identity:        validatorIdentity,
-			AuthorizedVoter: validatorAuthorizedVoter,
-			VoteAccount:     validatorVoteAccount,
-			HistoryDir:      blockstorePath,
-			EpochForSlot:    epochSchedule.GetEpoch,
-			SlotDuration:    blockprod.AlpenglowSlotDuration,
-			WaitToVoteSlot:  waitToVoteSlot,
-			ReadyToVote: func(slot uint64) bool {
-				wallSlot := global.WallClockSlot()
-				if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok {
-					wallSlot = liveSlot
-				}
-				return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots
-			},
-			Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer {
-				peers := make([]alpenglow.VotorPeer, 0, len(validators))
-				seen := make(map[solana.PublicKey]struct{}, len(validators))
-				for _, validator := range validators {
-					if validator.Stake == 0 || validator.NodePubkey == identityPubkey {
-						continue
-					}
-					addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey)
-					if !ok {
-						continue
-					}
-					if _, duplicate := seen[validator.NodePubkey]; duplicate {
-						continue
-					}
-					seen[validator.NodePubkey] = struct{}{}
-					peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr})
-				}
-				return peers
-			},
-		}); err != nil {
-			klog.Fatalf("enable Alpenglow voting: %v", err)
-		}
 		broadcaster, err := turbine.NewTurbineBroadcaster(turbine.TurbineBroadcasterConfig{
 			Self:          solana.PrivateKey(validatorIdentity).PublicKey(),
 			Peers:         sharedGossip,
@@ -2744,6 +2712,50 @@ postBootstrap:
 			mlog.Log.Warnf("validator gossip TPU advertisement: %v", err)
 		}
 
+		// Bind and validate local transports before consuming the durable clean
+		// voting marker. Startup configuration failures must not force recovery.
+		identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey()
+		if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{
+			Identity:                  validatorIdentity,
+			AuthorizedVoter:           validatorAuthorizedVoter,
+			VoteAccount:               validatorVoteAccount,
+			HistoryDir:                blockstorePath,
+			ReservedHistory:           validatorReservedHistory,
+			InitializeVoteReservation: validatorInitializeReservation,
+			Genesis:                   solana.MustHashFromBase58(networkGenesisHash),
+			EpochForSlot:              epochSchedule.GetEpoch,
+			SlotDuration:              blockprod.AlpenglowSlotDuration,
+			WaitToVoteSlot:            waitToVoteSlot,
+			ReadyToVote: func(slot uint64) bool {
+				wallSlot := global.WallClockSlot()
+				if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok {
+					wallSlot = liveSlot
+				}
+				return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots
+			},
+			Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer {
+				peers := make([]alpenglow.VotorPeer, 0, len(validators))
+				seen := make(map[solana.PublicKey]struct{}, len(validators))
+				for _, validator := range validators {
+					if validator.Stake == 0 || validator.NodePubkey == identityPubkey {
+						continue
+					}
+					addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey)
+					if !ok {
+						continue
+					}
+					if _, duplicate := seen[validator.NodePubkey]; duplicate {
+						continue
+					}
+					seen[validator.NodePubkey] = struct{}{}
+					peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr})
+				}
+				return peers
+			},
+		}); err != nil {
+			klog.Fatalf("enable Alpenglow voting: %v", err)
+		}
+
 		rewardBuilder := rewardcerts.NewBuilder(rewardcerts.BuilderConfig{
 			RootSlot:    global.Slot,
 			BeforeBuild: consensusEngine.FlushAlpenglowRewardVotes,
@@ -2796,6 +2808,7 @@ postBootstrap:
 				}
 			},
 			ProductionParent: consensusEngine.AlpenglowBlockProductionParent,
+			CanSignSlot:      consensusEngine.AlpenglowCanSignLeaderSlot,
 			CurrentSlot: func() uint64 {
 				if slot, ok := consensusEngine.AlpenglowLiveSlot(); ok {
 					return slot

cmd/mithril/node/vote_startup.go

@@ -0,0 +1,40 @@
+package node
+
+import (
+	"fmt"
+	"math"
+	"strconv"
+
+	"github.com/Overclock-Validator/mithril/pkg/alpenglow"
+	"github.com/Overclock-Validator/mithril/pkg/config"
+	"github.com/spf13/cobra"
+)
+
+func configuredWaitToVoteSlot(cmd *cobra.Command) (uint64, error) {
+	if flag := cmd.Flags().Lookup("wait-to-vote-slot"); flag != nil && flag.Changed {
+		return cmd.Flags().GetUint64("wait-to-vote-slot")
+	}
+	const key = "validator.wait_to_vote_slot"
+	if !config.IsSet(key) {
+		return 0, nil
+	}
+	// Unlike GetUint64, parsing explicitly must not turn an invalid operator
+	// cutoff into zero and silently remove the requested voting restriction.
+	slot, err := strconv.ParseUint(config.GetString(key), 10, 64)
+	if err != nil {
+		return 0, fmt.Errorf("%s must be an unsigned 64-bit slot: %w", key, err)
+	}
+	return slot, nil
+}
+
+// The operator cutoff can postpone voting but cannot weaken the existing
+// startup guard. Equality permits voting, subject to all other Votor checks.
+func effectiveWaitToVoteSlot(startupWallSlot, configured uint64) uint64 {
+	automatic := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots
+	if automatic <= math.MaxUint64-2*alpenglow.LeaderWindowSlots {
+		automatic += 2 * alpenglow.LeaderWindowSlots
+	} else {
+		automatic = math.MaxUint64
+	}
+	return max(automatic, configured)
+}

cmd/mithril/node/vote_startup_test.go

@@ -0,0 +1,73 @@
+package node
+
+import (
+	"math"
+	"strings"
+	"testing"
+
+	"github.com/Overclock-Validator/mithril/pkg/config"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/require"
+)
+
+func TestConfiguredWaitToVoteSlot(t *testing.T) {
+	for _, tc := range []struct {
+		name, toml, cli string
+		want            uint64
+		invalid         bool
+	}{
+		{name: "default"},
+		{name: "toml", toml: "wait_to_vote_slot = 1234", want: 1234},
+		{name: "cli wins", toml: "wait_to_vote_slot = 1234", cli: "5678", want: 5678},
+		{name: "explicit zero wins", toml: "wait_to_vote_slot = 1234", cli: "0"},
+		{name: "maximum CLI", cli: "18446744073709551615", want: math.MaxUint64},
+		{name: "negative TOML", toml: "wait_to_vote_slot = -1", invalid: true},
+		{name: "fractional TOML", toml: "wait_to_vote_slot = 1.5", invalid: true},
+		{name: "malformed TOML value", toml: `wait_to_vote_slot = "oops"`, invalid: true},
+		{name: "empty TOML value", toml: `wait_to_vote_slot = ""`, invalid: true},
+		{name: "overflow TOML value", toml: `wait_to_vote_slot = "18446744073709551616"`, invalid: true},
+		{name: "negative CLI", cli: "-1", invalid: true},
+		{name: "overflow CLI", cli: "18446744073709551616", invalid: true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			viper.Reset()
+			t.Cleanup(viper.Reset)
+			config.ApplyDefaults(viper.GetViper())
+			viper.SetConfigType("toml")
+			require.NoError(t, viper.ReadConfig(strings.NewReader("[validator]\n"+tc.toml)))
+			cmd := &cobra.Command{}
+			cmd.Flags().Uint64("wait-to-vote-slot", 0, "")
+			var err error
+			if tc.cli != "" {
+				err = cmd.Flags().Set("wait-to-vote-slot", tc.cli)
+			}
+			var got uint64
+			if err == nil {
+				got, err = configuredWaitToVoteSlot(cmd)
+			}
+			if tc.invalid {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.Equal(t, tc.want, got)
+		})
+	}
+	require.NotNil(t, Run.Flags().Lookup("wait-to-vote-slot"))
+}
+
+func TestEffectiveWaitToVoteSlot(t *testing.T) {
+	for _, tc := range []struct{ startup, configured, want uint64 }{
+		{100, 0, 108},
+		{103, 0, 108},
+		{103, 104, 108},
+		{103, 108, 108},
+		{103, 123, 123}, // Operator cutoff need not align with a leader window.
+		{103, math.MaxUint64, math.MaxUint64},
+		{math.MaxUint64 - 7, 0, math.MaxUint64},
+		{math.MaxUint64, 0, math.MaxUint64},
+	} {
+		require.Equal(t, tc.want, effectiveWaitToVoteSlot(tc.startup, tc.configured), "%+v", tc)
+	}
+}

config.example.toml

@@ -328,6 +328,16 @@ name = "mithril"
     # Signature-verification workers (0 = GOMAXPROCS).
     tpu_sigverify_workers = 0
 
+    # Optional minimum slot for NEW votes (inclusive), also --wait-to-vote-slot.
+    # Useful when rejoining after recovery. CLI overrides this setting.
+    # Zero adds no operator cutoff; the automatic startup cutoff and normal
+    # consensus checks still apply. A lower value cannot bypass those checks.
+    # Replay/repair continue while waiting. Previously recorded, authenticated
+    # votes can still be restored/rebroadcast under the existing recovery rules.
+    # This does not coordinate a cluster restart or wait for supermajority,
+    # and does not allow resetting a corrupt vote-history file.
+    wait_to_vote_slot = 0
+
 # ============================================================================
 # [consensus] - Alpenglow Consensus
 # ============================================================================