diff --git a/.github/workflows/go_build.yml b/.github/workflows/go_build.yml index 50383467f..4fa7306c9 100644 --- a/.github/workflows/go_build.yml +++ b/.github/workflows/go_build.yml @@ -17,3 +17,31 @@ jobs: - name: Build run: go build -v ./cmd/mithril + + regression-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + GOMAXPROCS: "2" + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.26.4 + + - name: Voting, checkpoint, streaming and scheduler race regressions + # Run the complete affected package suites, including subprocess crash + # recovery and cancellation tests. The independent sealevel suite has + # known base-branch failures documented in the validation report. + run: >- + go test -race -p 2 -count=1 + ./pkg/alpenglow ./pkg/consensus ./pkg/replay + ./pkg/turbine ./pkg/sigverify ./pkg/blockprod/... + ./cmd/mithril/node ./cmd/mithril/configcmd + + - name: Vote-program deque ownership race regression + run: go test -race -count=1 ./pkg/sealevel -run '^TestProcessNewVoteStateOwnsRetainedDeque$' diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index c452d494e..1f6a21d8e 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -103,6 +103,9 @@ var ( validatorTPUQUICBind string validatorAdvertisedIP string validatorSigverifyWorkers int + validatorWaitToVoteSlot uint64 + validatorReservedHistory bool + validatorInitializeReservation bool // Mode thresholds blockNearTipThreshold int // Enter near-tip when gap <= this @@ -547,6 +550,9 @@ func init() { Run.Flags().StringVar(&validatorTPUQUICBind, "tpu-quic-bind-addr", "", "Validator TPU QUIC listen address (default 0.0.0.0:8004)") Run.Flags().StringVar(&validatorAdvertisedIP, "validator-advertised-ip", "", "Public IP advertised for validator TPU QUIC") Run.Flags().IntVar(&validatorSigverifyWorkers, "tpu-sigverify-workers", 0, "TPU signature verification workers (0 = GOMAXPROCS)") + Run.Flags().BoolVar(&validatorReservedHistory, "reserved-vote-history", false, "Use durable signing reservations with unsynchronized per-vote history writes") + Run.Flags().BoolVar(&validatorInitializeReservation, "initialize-vote-reservation", false, "Enroll complete synchronous vote history in reserved mode (one-time migration)") + Run.Flags().Uint64Var(&validatorWaitToVoteSlot, "wait-to-vote-slot", 0, "Do not cast new votes below this slot; the automatic startup cutoff still applies (0 = automatic only)") // [tuning] section flags Run.Flags().Uint64Var(¶mArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)") @@ -639,6 +645,11 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { if err := config.InitConfig(); err != nil { return err } + if slot, err := configuredWaitToVoteSlot(cmd); err != nil { + return err + } else { + validatorWaitToVoteSlot = slot + } // Check if a CLI flag was explicitly set by the user flagChanged := func(name string) bool { @@ -2642,52 +2653,9 @@ postBootstrap: } global.SeedWallClockSlot(wallClockSeed) startupWallSlot := global.WallClockSlot() - waitToVoteSlot := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots - if waitToVoteSlot <= math.MaxUint64-2*alpenglow.LeaderWindowSlots { - waitToVoteSlot += 2 * alpenglow.LeaderWindowSlots - } else { - waitToVoteSlot = math.MaxUint64 - } - mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d wait_to_vote=%d", startupWallSlot, waitToVoteSlot) + waitToVoteSlot := effectiveWaitToVoteSlot(startupWallSlot, validatorWaitToVoteSlot) + mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d configured_wait_to_vote=%d wait_to_vote=%d", startupWallSlot, validatorWaitToVoteSlot, waitToVoteSlot) - identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey() - if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{ - Identity: validatorIdentity, - AuthorizedVoter: validatorAuthorizedVoter, - VoteAccount: validatorVoteAccount, - HistoryDir: blockstorePath, - EpochForSlot: epochSchedule.GetEpoch, - SlotDuration: blockprod.AlpenglowSlotDuration, - WaitToVoteSlot: waitToVoteSlot, - ReadyToVote: func(slot uint64) bool { - wallSlot := global.WallClockSlot() - if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok { - wallSlot = liveSlot - } - return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots - }, - Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer { - peers := make([]alpenglow.VotorPeer, 0, len(validators)) - seen := make(map[solana.PublicKey]struct{}, len(validators)) - for _, validator := range validators { - if validator.Stake == 0 || validator.NodePubkey == identityPubkey { - continue - } - addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey) - if !ok { - continue - } - if _, duplicate := seen[validator.NodePubkey]; duplicate { - continue - } - seen[validator.NodePubkey] = struct{}{} - peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr}) - } - return peers - }, - }); err != nil { - klog.Fatalf("enable Alpenglow voting: %v", err) - } broadcaster, err := turbine.NewTurbineBroadcaster(turbine.TurbineBroadcasterConfig{ Self: solana.PrivateKey(validatorIdentity).PublicKey(), Peers: sharedGossip, @@ -2744,6 +2712,50 @@ postBootstrap: mlog.Log.Warnf("validator gossip TPU advertisement: %v", err) } + // Bind and validate local transports before consuming the durable clean + // voting marker. Startup configuration failures must not force recovery. + identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey() + if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{ + Identity: validatorIdentity, + AuthorizedVoter: validatorAuthorizedVoter, + VoteAccount: validatorVoteAccount, + HistoryDir: blockstorePath, + ReservedHistory: validatorReservedHistory, + InitializeVoteReservation: validatorInitializeReservation, + Genesis: solana.MustHashFromBase58(networkGenesisHash), + EpochForSlot: epochSchedule.GetEpoch, + SlotDuration: blockprod.AlpenglowSlotDuration, + WaitToVoteSlot: waitToVoteSlot, + ReadyToVote: func(slot uint64) bool { + wallSlot := global.WallClockSlot() + if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok { + wallSlot = liveSlot + } + return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots + }, + Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer { + peers := make([]alpenglow.VotorPeer, 0, len(validators)) + seen := make(map[solana.PublicKey]struct{}, len(validators)) + for _, validator := range validators { + if validator.Stake == 0 || validator.NodePubkey == identityPubkey { + continue + } + addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey) + if !ok { + continue + } + if _, duplicate := seen[validator.NodePubkey]; duplicate { + continue + } + seen[validator.NodePubkey] = struct{}{} + peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr}) + } + return peers + }, + }); err != nil { + klog.Fatalf("enable Alpenglow voting: %v", err) + } + rewardBuilder := rewardcerts.NewBuilder(rewardcerts.BuilderConfig{ RootSlot: global.Slot, BeforeBuild: consensusEngine.FlushAlpenglowRewardVotes, @@ -2796,6 +2808,7 @@ postBootstrap: } }, ProductionParent: consensusEngine.AlpenglowBlockProductionParent, + CanSignSlot: consensusEngine.AlpenglowCanSignLeaderSlot, CurrentSlot: func() uint64 { if slot, ok := consensusEngine.AlpenglowLiveSlot(); ok { return slot diff --git a/cmd/mithril/node/vote_startup.go b/cmd/mithril/node/vote_startup.go new file mode 100644 index 000000000..547e55194 --- /dev/null +++ b/cmd/mithril/node/vote_startup.go @@ -0,0 +1,40 @@ +package node + +import ( + "fmt" + "math" + "strconv" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/spf13/cobra" +) + +func configuredWaitToVoteSlot(cmd *cobra.Command) (uint64, error) { + if flag := cmd.Flags().Lookup("wait-to-vote-slot"); flag != nil && flag.Changed { + return cmd.Flags().GetUint64("wait-to-vote-slot") + } + const key = "validator.wait_to_vote_slot" + if !config.IsSet(key) { + return 0, nil + } + // Unlike GetUint64, parsing explicitly must not turn an invalid operator + // cutoff into zero and silently remove the requested voting restriction. + slot, err := strconv.ParseUint(config.GetString(key), 10, 64) + if err != nil { + return 0, fmt.Errorf("%s must be an unsigned 64-bit slot: %w", key, err) + } + return slot, nil +} + +// The operator cutoff can postpone voting but cannot weaken the existing +// startup guard. Equality permits voting, subject to all other Votor checks. +func effectiveWaitToVoteSlot(startupWallSlot, configured uint64) uint64 { + automatic := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots + if automatic <= math.MaxUint64-2*alpenglow.LeaderWindowSlots { + automatic += 2 * alpenglow.LeaderWindowSlots + } else { + automatic = math.MaxUint64 + } + return max(automatic, configured) +} diff --git a/cmd/mithril/node/vote_startup_test.go b/cmd/mithril/node/vote_startup_test.go new file mode 100644 index 000000000..d86a02528 --- /dev/null +++ b/cmd/mithril/node/vote_startup_test.go @@ -0,0 +1,73 @@ +package node + +import ( + "math" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestConfiguredWaitToVoteSlot(t *testing.T) { + for _, tc := range []struct { + name, toml, cli string + want uint64 + invalid bool + }{ + {name: "default"}, + {name: "toml", toml: "wait_to_vote_slot = 1234", want: 1234}, + {name: "cli wins", toml: "wait_to_vote_slot = 1234", cli: "5678", want: 5678}, + {name: "explicit zero wins", toml: "wait_to_vote_slot = 1234", cli: "0"}, + {name: "maximum CLI", cli: "18446744073709551615", want: math.MaxUint64}, + {name: "negative TOML", toml: "wait_to_vote_slot = -1", invalid: true}, + {name: "fractional TOML", toml: "wait_to_vote_slot = 1.5", invalid: true}, + {name: "malformed TOML value", toml: `wait_to_vote_slot = "oops"`, invalid: true}, + {name: "empty TOML value", toml: `wait_to_vote_slot = ""`, invalid: true}, + {name: "overflow TOML value", toml: `wait_to_vote_slot = "18446744073709551616"`, invalid: true}, + {name: "negative CLI", cli: "-1", invalid: true}, + {name: "overflow CLI", cli: "18446744073709551616", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + viper.Reset() + t.Cleanup(viper.Reset) + config.ApplyDefaults(viper.GetViper()) + viper.SetConfigType("toml") + require.NoError(t, viper.ReadConfig(strings.NewReader("[validator]\n"+tc.toml))) + cmd := &cobra.Command{} + cmd.Flags().Uint64("wait-to-vote-slot", 0, "") + var err error + if tc.cli != "" { + err = cmd.Flags().Set("wait-to-vote-slot", tc.cli) + } + var got uint64 + if err == nil { + got, err = configuredWaitToVoteSlot(cmd) + } + if tc.invalid { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + require.NotNil(t, Run.Flags().Lookup("wait-to-vote-slot")) +} + +func TestEffectiveWaitToVoteSlot(t *testing.T) { + for _, tc := range []struct{ startup, configured, want uint64 }{ + {100, 0, 108}, + {103, 0, 108}, + {103, 104, 108}, + {103, 108, 108}, + {103, 123, 123}, // Operator cutoff need not align with a leader window. + {103, math.MaxUint64, math.MaxUint64}, + {math.MaxUint64 - 7, 0, math.MaxUint64}, + {math.MaxUint64, 0, math.MaxUint64}, + } { + require.Equal(t, tc.want, effectiveWaitToVoteSlot(tc.startup, tc.configured), "%+v", tc) + } +} diff --git a/config.example.toml b/config.example.toml index 7e3833087..f46e78549 100644 --- a/config.example.toml +++ b/config.example.toml @@ -328,6 +328,16 @@ name = "mithril" # Signature-verification workers (0 = GOMAXPROCS). tpu_sigverify_workers = 0 + # Optional minimum slot for NEW votes (inclusive), also --wait-to-vote-slot. + # Useful when rejoining after recovery. CLI overrides this setting. + # Zero adds no operator cutoff; the automatic startup cutoff and normal + # consensus checks still apply. A lower value cannot bypass those checks. + # Replay/repair continue while waiting. Previously recorded, authenticated + # votes can still be restored/rebroadcast under the existing recovery rules. + # This does not coordinate a cluster restart or wait for supermajority, + # and does not allow resetting a corrupt vote-history file. + wait_to_vote_slot = 0 + # ============================================================================ # [consensus] - Alpenglow Consensus # ============================================================================ diff --git a/docs/alpenglow_branch_engine.md b/docs/alpenglow_branch_engine.md index c155881fb..3171d0c1d 100644 --- a/docs/alpenglow_branch_engine.md +++ b/docs/alpenglow_branch_engine.md @@ -160,6 +160,31 @@ mixed (heterogeneous-client) or Mithril-only cluster identically: timeouts, vote signing/transmission, durable vote-history persistence, and standstill participation. +## Replay-observer diagnostics + +The observer retains certificate history for deduplication and match/mismatch +reporting. A separate bounded index contains only retained, block-bearing +certificates that have not yet been reconciled against replay. Reconciliation +removes an entry after either a match or mismatch; eviction removes it together +with the historical certificate. Hashless/skipped replay cannot reconcile a +block-bearing certificate. Pending counts and age/window statistics retain the +same semantics, but scan unresolved entries rather than completed history. + +This index is disposable, process-local diagnostic state. It neither authorizes +votes nor substitutes for verified certificates, the chain tracker's finality +checks, durable signing bounds, vote history, or checkpoint recovery. Those +checks and persistence contracts are unchanged. + +`BenchmarkObserverEmptyReplay` measures observer work for an empty block, with +or without four preceding skipped slots, against 4,096 retained certificates. +It covers 0, 32, and 4,096 unresolved entries. On Ryzen 9700X (GOMAXPROCS=8, +three 300 ms runs), median time for the four-skips-plus-empty case with 32 +unresolved entries was 686.4 µs before the index and 1.87 µs afterward. With +all 4,096 entries unresolved it was 380.5 → 159.3 µs. These are component +benchmarks; they exclude execution, certificate cryptography, network delivery, +and end-to-end FAST inclusion. Live comparisons must account for observer +history warming after a restart and different leader/skip patterns. + ## What this proves — and does not The certificate layer proves which block *data* the cluster settled on. In diff --git a/docs/certificate-processing-evidence.md b/docs/certificate-processing-evidence.md new file mode 100644 index 000000000..c7b520914 --- /dev/null +++ b/docs/certificate-processing-evidence.md @@ -0,0 +1,14 @@ +# Certificate Processing: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc) +(tag `review-evidence-20260916-certificate-processing`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/docs/certpool-offlock.md b/docs/certpool-offlock.md new file mode 100644 index 000000000..646d3e70b --- /dev/null +++ b/docs/certpool-offlock.md @@ -0,0 +1,78 @@ +# Incoming vote verification outside the pool lock + +The incoming vote pool previously held one mutex while checking BLS batches and folding signatures into aggregates. A 45-second trace on the live Zen 5 validator observed a 9.49 ms p95 acquisition wait and a 57.68 ms maximum across 34,580 normally completed AddVote calls. This blocked other incoming votes and readers; durable pruning could also wait for this lock while holding the consensus output mutex. + +The change retains one expensive BLS batch at a time using a separate verification mutex, while releasing the pool-state mutex around cryptography. One caller owns processing for each slot. Ordinary arrivals may join that slot's pending maps while verification runs, and the owner revisits state after those arrivals. Admission that needs authentication to resolve quota pressure or competing signatures waits instead of weakening bounds or first-packet-poisoning checks. + +Candidates remain pending until verification finishes, so in-flight votes count against admission limits and exact duplicates consume no additional space. Only selected candidates are removed on completion. Pruning and eviction may remove a slot during verification; pointer identity checks prevent stale work from resurrecting it or releasing accounting twice. If the epoch lookup, installed validator-set identity or shred version changes, the old slot state is retired instead of mixing bindings. Normal engine validator sets are immutable within an epoch. + +Each completed verified batch publishes promptly, outside both mutexes. New arrivals do not defer an authenticated quorum until the slot stops receiving votes. Reward-footer flushing waits for the active owner, drains relevant pending votes, and honors the publication barrier. Verified stake reads preserve freshness by waiting for that slot's owner. Snapshot counters and durable pruning do not wait for cryptography. + +Point reuse is included: aggregation uses already-verified signature points, failed batches subdivide parsed members, and the unused tally public-key aggregate is removed. Randomized coefficients, signature checks, stake thresholds, equivocation budgets and paired-vote disjointness remain. + +## Lock ownership + +`verifyAndFoldTallyWithLockReleased` requires the pool lock on entry and returns +with it held, including early exits. It releases the lock during crypto and +revalidates slot and validator bindings after reacquiring it. The caller owns +that slot's processing marker. `finishSlotAndUnlock` consumes lock ownership: +it releases the lock, emits certificates and returns unlocked. Callers must not +pair it with a deferred unlock. + +## Bounded multi-scalar aggregation + +With more than one Go execution thread, batches of at least 16 parsed votes +use gnark `MultiExp` for each weighted +public-key/signature sum. G1 and G2 run sequentially with `NbTasks: 1`; the +existing pool verification mutex still admits one expensive batch at a time. +Smaller batches keep the scalar loop because bucket setup costs more than it +saves, particularly during failed-batch subdivision and two-candidate checks. +Single-thread configurations also retain the scalar path: native contention +tests found that MultiExp task handoffs could increase certificate wall time +there, despite reducing arithmetic. + +Each member still receives a fresh, independent, nonzero coefficient sampled +from the full scalar field. Its public key and signature use the same +coefficient. Entropy/aggregation errors retain the individual-verification +fallback. Parsing, subgroup/infinity checks, failed-batch subdivision, +publication and stake accounting are unchanged. + +### Native staging validation + +AMD Ryzen 7 9700X, Go 1.26.4, GOMAXPROCS=2, Nice 15, two-core CPU quota, with +the normal validator workload still running. The baseline restores the scalar +implementation from the preceding certificate split head (`395e4566`) with +identical fixtures. Baseline/candidate/candidate/baseline runs were followed by +a final check after adding the single-thread safeguard. Full-fold ranges: + +| Votes in fold | Scalar baseline | Final implementation | +| --- | ---: | ---: | +| 32 | 7.385–7.386 ms | 4.238–4.344 ms | +| 64 | 14.217–14.280 ms | 6.800–7.135 ms | + +A controlled scheduling experiment runs 64 valid votes every 200 ms alongside +4,096 repeated transfer executions. With two Go execution threads, the final +alternating native comparison reduced certificate processing from +16.25–16.39 ms to 8.43–10.22 ms. Execution results were noisy: 13.80–14.38 ms +baseline versus 13.84–17.56 ms candidate. The earlier native comparison and the +local comparison showed no execution regression, but the slower final sample +is retained; the shared host does not establish absence of interference. +This workload excludes account commits, network delivery and vote persistence. + +The initial prototype's single-thread certificate latency could worsen while +execution improved slightly. The final implementation therefore keeps the +original scalar arithmetic whenever GOMAXPROCS is one. No worker-count or +verification-admission changes accompany this optimization. + +## Reproduce and validate + +Run `go test -race ./pkg/alpenglow` for concurrency, pending-budget, stale-binding, +invalid-share, equivocation and publication-barrier coverage. +Run `go test ./pkg/alpenglow -run '^$' -bench '^BenchmarkCertPool(WeightedPairing|FoldVerifiedBatch)$' -benchmem -benchtime=500ms -count=3` with the same fixture on each revision. + +The earlier concurrent 64-vote diagnostic reduced Snapshot p95 from +8.57–8.72 ms with point reuse alone to 0.008–0.038 ms with verification outside +the lock. This measures reader latency, not vote throughput. The diagnostic's +ns/op includes intentional sampling pauses. Historical component baselines, +live observations and full qualifications are preserved in the +[evidence archive](certificate-processing-evidence.md). diff --git a/docs/certpool-point-reuse.md b/docs/certpool-point-reuse.md new file mode 100644 index 000000000..cc2ac3a84 --- /dev/null +++ b/docs/certpool-point-reuse.md @@ -0,0 +1,18 @@ +# Reuse verified BLS signature points + +Incoming verification returns parsed, verified members and reuses their signature +points when folding the tally. Failed aggregate checks subdivide parsed members +instead of reparsing each subset. Individual verification returns the same member +representation. Installed validator sets retain their parsed public keys. + +The unused tally public-key sum is removed. Randomized verification still builds +its required weighted public-key sum with independent full-field coefficients. +Subgroup/infinity checks, invalid-share rejection, duplicate/equivocation checks, +stake accounting and paired-vote disjointness remain enforced. + +Point ownership and message association are covered by differential tests against +individual verification, including malformed signatures, invalid ranks and wrong +payloads. See [pool concurrency and aggregation](certpool-offlock.md) for the +current lock contract, combined implementation and benchmark method. +The isolated point-reuse experiment is preserved in the +[historical evidence](certificate-processing-evidence.md). diff --git a/docs/reserved-vote-history.md b/docs/reserved-vote-history.md new file mode 100644 index 000000000..bd94e2e57 --- /dev/null +++ b/docs/reserved-vote-history.md @@ -0,0 +1,162 @@ +# Voting persistence and crash recovery + +## Intended guarantee + +A crash must not lead to conflicting externally published votes or reserved-mode +leader actions because the validator forgot its earlier local decisions. The design permits +loss of recent detailed history and sacrifices voting availability when its +completeness is uncertain. It does not promise immediate restart voting, that +every signed vote reaches disk, or recovery from rolled-back safety files. +Normal anti-equivocation, execution, parent and validator-binding checks remain +necessary; this is a persistence contract, not a proof of the entire protocol. + +The failure model includes process termination and host/power failure, provided +successful file and directory syncs survive, the current safety files are +preserved, and only one fenced owner uses the signing identity. Software tests +exercise the recovery decisions; they do not qualify actual storage against +power loss. Valid signatures on saved files establish integrity, not freshness. + +## Two different publication guarantees + +| Mode | Required before a vote can escape to the pool/network | What a restart may trust | +| --- | --- | --- | +| Default synchronous history | Exact validated history is written, file-synced, renamed and directory-synced before local pool admission or network enqueue. | Retained exact decisions and their rooted boundary, subject to normal restoration checks. | +| Opt-in reserved history | The vote's slot is covered by an acknowledged durable reservation before signing. Its exact history snapshot is prepared and queued before publication, without waiting for per-vote I/O. | The startup reservation bound, unless a separately validated clean-history seal proves exact retained history. | + +In synchronous mode, BLS bytes may be computed privately in RAM before history +is synced. The guarantee is **persist before publication**, including local +pool admission because it can publish a certificate. In reserved mode, a +successful queue submission, background rename, or `written` counter is **not** +a durable acknowledgement of that vote. Replacing a whole file atomically is +not the same as making its bytes/directory entry survive power loss. + +Both modes retain complete decisions in memory while running and prune them +only through the normal verified-root rules. The synchronous history guarantee +covers votes; it is not a complete journal of produced leader blocks. The +additional leader reservation barrier applies only in reserved mode. + +## Reserved-mode restart rule + +Let **H** be the reservation's `Through` value loaded at startup, **F** the +verified finality/checkpoint floor, and **S** a proposed signing slot. H bounds +what the previous process *might* have signed; it is not its last actual vote. + +Without a validated clean seal, every vote type and historical local-vote +restoration must obey all of these conditions: + +- **S > H**: never re-sign in the uncertain range during this run, even when + an older history file contains that particular vote. +- **F >= H**: do not sign above the range until verified finality/checkpoint + state has reached its end. F equal to H is sufficient; S equal to H is not. +- **S <= the current acknowledged reservation**, plus all ordinary protocol, + live-joining and configured minimum-slot checks. + +The startup recovery bound stays fixed during the run. Background renewal may +raise the current signing allowance; it does not move the recovery target. +Newly received blocks, elapsed wall time, an RPC tip, replay progress alone, and +`--wait-to-vote-slot` cannot substitute for verified finality/checkpoint state. +The recovery wait can be indefinite if the cluster halts below H. + +For example, suppose votes through 1,015 escaped, detailed history survives only +through 1,012, and the durable reservation is 1,032. After an unclean restart, +slots <= 1,032 remain forbidden. Slot 1,033 is also forbidden while F < 1,032. +Once F >= 1,032, it can pass the recovery gate only after an acknowledged grant +covers 1,033 and all normal voting checks pass. Seeing a new block after restart +does not by itself meet these conditions. + +## Clean shutdown is a durable protocol + +1. Stop/join the voter and leader producer. Halt/join the reservation worker; + drain/join the history writer so an older rename cannot overwrite the seal. +2. Require no latched safety fault or history-write failure, no unresolved + reservation-write uncertainty, and verified recovery through the startup H. +3. Sync exact retained history, including the verified rooted boundary. +4. Sync a reservation record containing the digest of that exact history. + +A successful process exit, a signal handler running, or an attempted final save +is not sufficient. Startup must load and validate the history and reservation, +match the clean digest, and **durably consume the clean marker with a dirty +successor before new vote/leader signing or new history decisions**. A later crash uses +the reservation again, even if the old history still looks valid. + +| Restart state | Vote recovery | Leader recovery | +| --- | --- | --- | +| Valid history and reservation, no matching clean seal | Enforce S > H and F >= H, including restoration. | Enforce S > H and F >= H. | +| Valid matching clean seal, successfully consumed | Resume using exact retained voting decisions and ordinary checks; no additional vote quarantine through H. | Still enforce S > H and F >= H: vote history does not enumerate every block that may have been signed. | +| Missing, corrupt, unreadable or incompatible enrolled state | Refuse automatic reset/startup; do not infer safety from an RPC tip. | Same refusal. | + +Errors during sealing do not authorize treating the session as clean; startup +must validate whichever durable record survived. A failed reservation write +may have reached storage despite its error. Running signers retain only the +previous acknowledged allowance; a later monotonic successful write can resolve +that uncertainty. An unresolved error prevents deliberately sealing clean. + +## Lifetime, storage and enrollment + +The reservation grants up to 32 slots beyond the requested slot and renews when +16 or fewer remain. Only successful file-and-directory sync acknowledgement +publishes new permission. Exhaustion pauses signing while replay/verification +continue. Repeated restarts without new grants do not advance the bound. + +Detailed history uses one ordered writer with one in-flight and at most one +newer pending complete snapshot. New complete snapshots may supersede unwritten +ones. Validation/encoding/signing of the snapshot remain on the voter goroutine; +only file I/O is asynchronous. Writer errors latch a safety fault and stop +voting. An already in-flight vote is still covered by its durable reservation. + +Preserve both `vote_history-.mithril.json` and +`vote_reservation-.mithril.json` independently of AccountsDB snapshots. +The checkpoint encoding cache is only an encoding optimization; it is not the +vote journal or a replacement for the reservation. Rolling back AccountsDB or +an application binary must not roll back either signing-safety file. + +The directory lock only excludes concurrent owners of the same history path. +It cannot fence copies of the identity on other hosts/paths. Signed records and +generation numbers cannot detect an operator restoring an old valid pair of +safety files. Media loss, stale safety-file restoration, dishonest sync behavior +and compromised/copied signing keys are outside this automatic recovery contract. + +Use `--reserved-vote-history` to opt in; default persistence remains synchronous. +First enrollment additionally requires `--initialize-vote-reservation`, exclusive +identity ownership and complete synchronous history from the stopped previous +writer. An empty directory is appropriate only for a previously unused identity. +The software cannot distinguish that case from deleting both files for an old +identity; initialization is not a safe disaster-recovery reset. Remove the +initialization flag afterward. These are CLI flags, not TOML settings. + +Enrolled history uses version 2, rejected by older binaries that do not enforce +the reservation. Missing/corrupt enrolled state or a changed identity, vote +account, authorized voter, genesis or shred version must not be automatically +re-enrolled. Disabling the flag or deleting files is not a supported downgrade. +Transport binding/validation runs before the clean marker is consumed. + +## Live admission is separate from restart recovery + +The live vote admission floor uses retained consensus-pool state and the history +root, not the newest finalization certificate. Replay can still contribute a +notarization within the bounded 16-slot retained tail before later certificates +are collected; finalized retained slots can also receive skips. Durable-root +pruning is ordered behind completed replay events on the voter. These admission +changes also apply to synchronous mode. They do not weaken reserved-mode F >= H. +`--wait-to-vote-slot N` adds an inclusive minimum; it cannot bypass recovery, +execution, retained-root or parent checks. + +## Evidence and limits + +Existing tests map the recovery contract to these cases: + +| Contract | Tests | +| --- | --- | +| Lost valid history suffix; fixed bound across repeated crashes | `TestReservedVotingLostHistorySuffixAndRepeatedCrash` | +| All five vote types, restoration and leader gates at H/H+1 | `TestReservedVotingEverySignatureTypeAtBound` | +| Clean-marker consumption and stricter leader restart | `TestReservedVotingCleanMarkerConsumedBeforeSigning` | +| Digest mismatch, missing/corrupt/domain-mismatched state | `TestReservedCleanDigestMismatchUsesCrashRecovery`, `TestReservedVotingRejectsMissingCorruptOrWrongDomain` | +| Pending/uncertain sync cannot authorize signing | `TestSigningReservationUnacknowledgedSyncCannotAuthorize`, `TestSigningReservationUncertainWriteSurvivesRestart` | +| Writer drain, failure and lost pending snapshots | `TestAsyncHistoryBlockedWriteDoesNotDelayVotesAndCleanCloseDrains`, `TestAsyncHistoryFailureStopsVoterAndPreventsCleanMarker`, `TestAsyncHistoryProcessCrashLosesPendingSnapshots` | + +The subprocess-kill test exercises lost in-flight/pending application snapshots; +it does not power-cycle a host or prove filesystem durability. Fault-injection +and older-history fixtures check the algorithm's decisions under the stated +storage contract. This is not a formal consensus proof or mainnet storage +qualification. Persistence benchmarks likewise do not establish replay or FAST +performance. diff --git a/docs/vote-delivery-persistence-evidence.md b/docs/vote-delivery-persistence-evidence.md new file mode 100644 index 000000000..1ae1f5cfa --- /dev/null +++ b/docs/vote-delivery-persistence-evidence.md @@ -0,0 +1,14 @@ +# Vote Delivery Persistence: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/54b233ff0e27fb929644f7d53bf4a699cb590cd8) +(tag `review-evidence-20260916-vote-delivery-persistence`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/54b233ff0e27fb929644f7d53bf4a699cb590cd8/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/docs/votor-peer-isolation.md b/docs/votor-peer-isolation.md new file mode 100644 index 000000000..061b0f80a --- /dev/null +++ b/docs/votor-peer-isolation.md @@ -0,0 +1,99 @@ +# Votor outbound peer isolation + +A stalled QUIC peer could previously occupy all 32 shared send/connect workers. +quic-go's `SendDatagram` blocks when its 32-frame connection queue is full. +Repeated jobs for that peer could therefore prevent votes reaching healthy +peers, even while the broadcaster reported zero queue drops. + +Each authenticated connection now owns one sender and a FIFO queue of at most +256 encoded messages. The encoded payload is immutable and shared across peer +queues. The existing bounded worker pool handles connection attempts only; +`VotorBroadcasterConfig.Workers` controls that pool. A blocked connection cannot +consume another peer's sender or a connection worker. + +A watchdog checks every 100 ms whether the active datagram's peer-queue wait +plus its current `SendDatagram` duration has reached one second. Occasional QUIC +PTO probes can free queue entries without proving delivery; they no longer +restart this budget for an old backlog. Dequeue also retires a connection before +feeding an already-one-second-old entry into QUIC, even if sends keep completing +between watchdog ticks. The effective active-send bound is one second from +fanout enqueue, plus up to one watchdog interval and runtime scheduling delay. + +Retirement closes that connection, wakes a blocked sender, discards its queued +copies and requests a bounded reconnect. This is an operational limit on local +queueing, not a consensus validity deadline or a remote-delivery guarantee. It +adds no per-send timer or goroutine. An idle connection is not expired merely +because its previous send had a long queue delay. + +## Failure and ordering semantics + +- The global `Enqueue` contract is unchanged: rejecting a newly signed message + from the global queue returns an error to the voting engine. +- Per-peer fanout remains best effort. A full peer queue rejects that peer's + newest copy, increments both `PeerQueueDrops` and the existing + `MessagesDropped` counter, and continues sending to other peers. +- Messages queued on a failed, removed or replaced connection are discarded and + counted in `PeerQueueDiscarded`. They are not transferred to a new connection + or address. A datagram expired at dequeue is counted in `PeerQueueDiscarded`; + a failed `SendDatagram` call is counted in `PeerSendErrors`. The watchdog and + dequeue path claim retirement under the sender mutex, counting one timeout. +- Every sender exit requests a bounded, deduplicated reconnect. This covers + both a remote-close notification and closure detected while dequeuing. Peer + departure, shutdown and a healthy replacement suppress obsolete requests; + normal remote closure no longer relies on the periodic reconciliation tick. +- Messages for disconnected peers increment `PeerSendsSkipped`, as before. + This change adds no automatic application-level retransmission. +- A single sender preserves local enqueue order within its connection. QUIC + datagrams themselves still do not guarantee arrival or ordering. +- Shutdown cancels connection attempts, closes the connections outside the + broadcaster mutex, drains their queues, and waits for sender exit. + +Signing decisions, persisted vote history, durable slot reservations, and +certificate validation are unchanged. + +## Observability + +The voting log adds `peer_queue_drops`, `peer_queue_discarded`, +`peer_send_timeouts`, and `peer_queue_max_delay`. These counters/high-water marks +survive reconnects. `PeerQueueMaxDelay` measures time from fanout enqueue to +sender dequeue, including entries retired before reaching QUIC; it does not +include time blocked inside `SendDatagram`. + +Voting snapshots also expose `broadcast_peer_queues` with each current peer's +identity, address, queue depth, time in the active send, last/max queue delay, +and queue drops. Durations in the JSON snapshot use nanoseconds. Per-connection +statistics disappear when that connection is replaced; totals remain available. +Successful `PeerSends` only means quic-go accepted the datagram, not that a +remote validator received it. + +## Regression coverage + +`TestVotorBroadcasterIsolatesBlockedPeer` establishes two real loopback QUIC +connections, then blackholes one UDP path. It observes the blocked +`datagramQueue.Add` stack and requires a healthy-peer vote to arrive within +250 ms while the failed connection is still open. It also covers peer queue +overflow, watchdog reconnection, fresh traffic after reconnect, shutdown with a +blocked sender, peer departure, and replacement of a blocked peer's address. +Its connection-retirement deadline is measured from the blackhole, with explicit +scheduler slack; the healthy-peer latency assertion remains 250 ms. + +Deterministic regressions reproduce a fresh send following an old queue wait, +remote closure while idle and while dequeuing, and an already-aged queue entry. +The reconnect fixture has no reconciliation loop, so a timer cannot hide a missed +reconnect trigger. The pre-fix failures and fresh validation are retained under +[historical evidence](https://github.com/Overclock-Validator/mithril/blob/54b233ff0e27fb929644f7d53bf4a699cb590cd8/docs/results/review-fixes/2026-09-15). + +The signature-verification config template and its tests now belong to the +streaming branch, which reads those settings. This standalone voting branch +retains its base branch's supported `tuning.sigverify_backend` template. + +```sh +go test -race ./pkg/alpenglow ./pkg/consensus -count=1 -timeout=180s +go vet ./pkg/alpenglow ./pkg/consensus +go build ./cmd/mithril +``` + +This is a transport regression, not a prediction of FAST score improvement. +The live validator needs a separately validated integrated build and matching +probe addresses before deployment; the PR checkout does not include every +change in the currently enrolled validator binary. diff --git a/pkg/alpenglow/broadcaster.go b/pkg/alpenglow/broadcaster.go index bd8e3ee7c..557ea997d 100644 --- a/pkg/alpenglow/broadcaster.go +++ b/pkg/alpenglow/broadcaster.go @@ -18,7 +18,7 @@ import ( const ( defaultVotorBroadcastQueue = 1024 - defaultVotorSendWorkers = 32 + defaultVotorConnectWorkers = 32 defaultVotorPeerJobQueue = 16384 defaultVotorPeerRefreshInterval = time.Second ) @@ -35,7 +35,8 @@ type VotorBroadcasterConfig struct { ShredVersion uint16 Peers VotorPeerSource QueueSize int - Workers int + // Workers bounds concurrent connection attempts. Sends are isolated per connection. + Workers int } type VotorBroadcasterStats struct { @@ -50,28 +51,25 @@ type VotorBroadcasterStats struct { ConnectionAttempts uint64 ConnectionErrors uint64 ConnectionJobsDropped uint64 + PeerQueueDrops uint64 + PeerQueueDiscarded uint64 + PeerSendTimeouts uint64 + PeerQueueMaxDelay time.Duration + PeerQueues []VotorPeerQueueStats LastPeerSendError string LastPeerSendErrorAt time.Time LastConnectionError string LastConnectionErrorAt time.Time } -type votorPeerJobKind uint8 - -const ( - votorPeerJobConnect votorPeerJobKind = iota - votorPeerJobSend -) - type votorPeerJob struct { - kind votorPeerJobKind - peer VotorPeer - payload []byte + peer VotorPeer } type votorConnection struct { - addr string - conn *quic.Conn + addr string + conn *quic.Conn + sender *votorPeerSender } type votorDial struct { @@ -109,6 +107,10 @@ type VotorBroadcaster struct { connectionAttempts atomic.Uint64 connectionErrors atomic.Uint64 connectionJobsDropped atomic.Uint64 + peerQueueDrops atomic.Uint64 + peerQueueDiscarded atomic.Uint64 + peerSendTimeouts atomic.Uint64 + peerQueueMaxDelay atomic.Int64 } func NewVotorBroadcaster(cfg VotorBroadcasterConfig) (*VotorBroadcaster, error) { @@ -122,7 +124,7 @@ func NewVotorBroadcaster(cfg VotorBroadcasterConfig) (*VotorBroadcaster, error) cfg.QueueSize = defaultVotorBroadcastQueue } if cfg.Workers <= 0 { - cfg.Workers = defaultVotorSendWorkers + cfg.Workers = defaultVotorConnectWorkers } certificate, err := newVotorQUICCertificate(cfg.Identity) if err != nil { @@ -151,7 +153,7 @@ func NewVotorBroadcaster(cfg VotorBroadcasterConfig) (*VotorBroadcaster, error) } b.wg.Add(2 + cfg.Workers) for range cfg.Workers { - go b.sendLoop() + go b.connectLoop() } go b.broadcastLoop() // Populate the desired set and queue the first bounded preconnects before @@ -199,71 +201,43 @@ func (b *VotorBroadcaster) broadcastLoop() { b.recordSendError(VotorPeer{}, fmt.Errorf("encode Votor message: %w", err)) continue } - peers, skipped := b.connectedPeers() + senders, skipped := b.connectedSenders() b.sendsSkipped.Add(uint64(skipped)) - for _, peer := range peers { - job := votorPeerJob{kind: votorPeerJobSend, peer: peer, payload: payload} - select { - case b.jobs <- job: - case <-b.done: - return - default: - b.dropped.Add(1) - } + job := votorDatagram{payload: payload, queuedAt: time.Now()} + for _, sender := range senders { + sender.enqueue(job) } } } } -func (b *VotorBroadcaster) sendLoop() { +// Connection attempts never occupy a peer's sender or delay connected peers. +func (b *VotorBroadcaster) connectLoop() { defer b.wg.Done() for { select { case <-b.done: return case job := <-b.jobs: - switch job.kind { - case votorPeerJobConnect: - b.connectPeer(job.peer.Identity) - case votorPeerJobSend: - if err := b.send(job.peer, job.payload); err != nil { - b.recordSendError(job.peer, err) - } else { - b.sends.Add(1) - } - } + b.connectPeer(job.peer.Identity) } } } -func (b *VotorBroadcaster) send(peer VotorPeer, payload []byte) error { - conn, ok := b.establishedConnection(peer) - if !ok { - b.queueConnect(peer.Identity) - return fmt.Errorf("send Votor datagram to %s (%s): no established connection", peer.Identity, peer.Addr) - } - err := conn.SendDatagram(payload) - if err == nil { - return nil - } - var tooLarge *quic.DatagramTooLargeError - if !errors.As(err, &tooLarge) { - b.dropConnection(peer.Identity, conn) - b.queueConnect(peer.Identity) - } - return fmt.Errorf("send Votor datagram to %s (%s): %w", peer.Identity, peer.Addr, err) -} - func (b *VotorBroadcaster) peerReconcileLoop() { defer b.wg.Done() ticker := time.NewTicker(defaultVotorPeerRefreshInterval) defer ticker.Stop() + watchdog := time.NewTicker(votorSendWatchInterval) + defer watchdog.Stop() for { select { case <-b.done: return case <-ticker.C: b.reconcilePeers() + case now := <-watchdog.C: + b.expirePeerSends(now) } } } @@ -305,9 +279,9 @@ func (b *VotorBroadcaster) reconcilePeers() { } } -func (b *VotorBroadcaster) connectedPeers() ([]VotorPeer, int) { +func (b *VotorBroadcaster) connectedSenders() ([]*votorPeerSender, int) { b.connMu.Lock() - peers := make([]VotorPeer, 0, len(b.desired)) + peers := make([]*votorPeerSender, 0, len(b.desired)) skipped := 0 for identity, peer := range b.desired { existing, connected := b.conns[identity] @@ -315,8 +289,7 @@ func (b *VotorBroadcaster) connectedPeers() ([]VotorPeer, int) { skipped++ continue } - peer.Addr = cloneUDPAddr(peer.Addr) - peers = append(peers, peer) + peers = append(peers, existing.sender) } b.connMu.Unlock() return peers, skipped @@ -349,7 +322,7 @@ func (b *VotorBroadcaster) queueConnectLocked(identity solana.PublicKey) { if _, queued := b.connectQueued[identity]; queued || b.dialing[identity] != nil { return } - job := votorPeerJob{kind: votorPeerJobConnect, peer: peer} + job := votorPeerJob{peer: peer} select { case b.jobs <- job: b.connectQueued[identity] = struct{}{} @@ -476,7 +449,12 @@ func (b *VotorBroadcaster) connection(peer VotorPeer) (*quic.Conn, error) { return existing.conn, nil } stale := b.conns[peer.Identity].conn - b.conns[peer.Identity] = votorConnection{addr: addr, conn: conn} + sender := &votorPeerSender{b: b, peer: peer, conn: conn, queue: make(chan votorDatagram, defaultVotorPeerSendQueue), done: make(chan struct{})} + b.conns[peer.Identity] = votorConnection{addr: addr, conn: conn, sender: sender} + // Close takes connMu before waiting, so no sender can be added after it + // observes the closed flag and drains the connection set. + b.wg.Add(1) + go sender.run() b.connMu.Unlock() if stale != nil { _ = stale.CloseWithError(0, "Votor peer address changed") @@ -499,12 +477,14 @@ func (b *VotorBroadcaster) Stats() VotorBroadcasterStats { } b.connMu.Lock() connections := 0 + peerQueues := make([]VotorPeerQueueStats, 0, len(b.conns)) for identity, existing := range b.conns { if existing.conn.Context().Err() != nil { delete(b.conns, identity) continue } connections++ + peerQueues = append(peerQueues, existing.sender.stats()) } desiredPeers := len(b.desired) pendingConnections := len(b.connectQueued) @@ -525,6 +505,11 @@ func (b *VotorBroadcaster) Stats() VotorBroadcasterStats { ConnectionAttempts: b.connectionAttempts.Load(), ConnectionErrors: b.connectionErrors.Load(), ConnectionJobsDropped: b.connectionJobsDropped.Load(), + PeerQueueDrops: b.peerQueueDrops.Load(), + PeerQueueDiscarded: b.peerQueueDiscarded.Load(), + PeerSendTimeouts: b.peerSendTimeouts.Load(), + PeerQueueMaxDelay: time.Duration(b.peerQueueMaxDelay.Load()), + PeerQueues: peerQueues, LastPeerSendError: lastSendError, LastPeerSendErrorAt: lastSendErrorAt, LastConnectionError: lastConnectionError, @@ -542,11 +527,15 @@ func (b *VotorBroadcaster) Close() error { close(b.done) b.connMu.Lock() clear(b.desired) + stale := make([]*quic.Conn, 0, len(b.conns)) for identity, existing := range b.conns { - _ = existing.conn.CloseWithError(0, "Votor broadcaster closed") + stale = append(stale, existing.conn) delete(b.conns, identity) } b.connMu.Unlock() + for _, conn := range stale { + _ = conn.CloseWithError(0, "Votor broadcaster closed") + } b.wg.Wait() }) return nil diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index 60b1882e3..33483cc10 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -5,8 +5,11 @@ import ( "crypto/sha256" "fmt" "math/big" + "runtime" "sync" + "sync/atomic" + "github.com/Overclock-Validator/gnark-crypto/ecc" bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" blsfr "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381/fr" "github.com/Overclock-Validator/mithril/pkg/mlog" @@ -31,7 +34,7 @@ const maxUnverifiedCandidatesPerRank = 2 // facts. Ingest (AddVote) only shape-checks, bounds buffering, and parks the // vote — it NEVER mutates dedupe, equivocation, disjointness, or tally state // that assembly or fork choice depends on. Those mutate only AFTER the BLS -// signature verifies (foldTallyLocked). This prevents a bogus vote for a +// signature verifies (verifyAndFoldTallyWithLockReleased). This prevents a bogus vote for a // victim rank from suppressing that validator's real vote (dedupe poisoning) // or forging equivocation evidence against an honest validator. // @@ -109,14 +112,12 @@ type tally struct { pending map[uint16]map[[sha256.Size]byte]VoteMessage // unverified candidates, by rank and signature verified map[uint16]struct{} // ranks folded into the aggregate aggSig bls12381.G2Affine // sum of verified signatures - aggPub bls12381.G1Affine // sum of verified pubkeys stake uint64 // verified stake } func newTally() *tally { t := &tally{pending: make(map[uint16]map[[sha256.Size]byte]VoteMessage), verified: make(map[uint16]struct{})} t.aggSig.SetInfinity() - t.aggPub.SetInfinity() return t } @@ -136,7 +137,14 @@ type tallyKey struct { } type poolSlot struct { - tallies map[tallyKey]*tally + // One caller owns folding for a slot while other callers may buffer votes. + // All fields, like the maps below, are protected by CertPool.mu. + processing bool + // Queued and active reward flushes take precedence over new arrivals/owners. + // A count preserves priority when multiple footer builders overlap. + flushWaiting int + dirty bool + tallies map[tallyKey]*tally // verifiedHash tracks the block hashes a rank has cast VERIFIED votes for, // per (rank, type), for equivocation/vote-budget enforcement. Populated only // after signature verification — never from raw ingest — so a bogus vote can @@ -166,16 +174,20 @@ type CertPool struct { // end and must not make downstream consensus decisions itself. verifiedVoteSink func(VerifiedVote) - mu sync.Mutex - epochForSlot func(slot uint64) uint64 - slots map[uint64]*poolSlot - emitted map[CertificateKey]struct{} - floor uint64 - liveSlot uint64 // trusted replay/observed watermark (NOT advanced by raw votes) - highestSlot uint64 // observability only: highest vote slot seen - totalPending int - equivocation []EquivocationEvidence - snap CertPoolSnapshot + mu sync.Mutex + workCond *sync.Cond + verificationMu sync.Mutex // Keep expensive batch concurrency bounded at one. + batchesVerified atomic.Uint64 + epochGeneration uint64 + epochForSlot func(slot uint64) uint64 + slots map[uint64]*poolSlot + emitted map[CertificateKey]struct{} + floor uint64 + liveSlot atomic.Uint64 // trusted replay/observed watermark (NOT advanced by raw votes) + highestSlot uint64 // observability only: highest vote slot seen + totalPending int + equivocation []EquivocationEvidence + snap CertPoolSnapshot publicationMu sync.Mutex publicationCond *sync.Cond @@ -214,6 +226,7 @@ func NewCertPool(cfg CertPoolConfig, verifier *CertificateVerifier, emit func(Ce publicationCompleted: make(map[uint64]struct{}), } p.publicationCond = sync.NewCond(&p.publicationMu) + p.workCond = sync.NewCond(&p.mu) return p } @@ -229,6 +242,7 @@ func (p *CertPool) SetVerifiedVoteSink(sink func(VerifiedVote)) { func (p *CertPool) SetEpochLookup(fn func(slot uint64) uint64) { p.mu.Lock() p.epochForSlot = fn + p.epochGeneration++ p.mu.Unlock() } @@ -236,21 +250,24 @@ func (p *CertPool) SetEpochLookup(fn func(slot uint64) uint64) { // (from replay progress / observed finality — never from raw votes, so an // attacker cannot slide the window forward). Monotonic. func (p *CertPool) NoteLiveSlot(slot uint64) { - p.mu.Lock() - if slot > p.liveSlot { - p.liveSlot = slot + // Replay must not wait for BLS verification under p.mu merely to announce + // progress. Concurrent trusted updates may arrive out of order. + for current := p.liveSlot.Load(); slot > current; current = p.liveSlot.Load() { + if p.liveSlot.CompareAndSwap(current, slot) { + return + } } - p.mu.Unlock() } // windowAnchorLocked is the trusted upper anchor of the live vote window: the // higher of the finalized floor and the replay-observed live slot. It is NOT // derived from raw votes, so ingest cannot advance it. func (p *CertPool) windowAnchorLocked() uint64 { - if p.floor > p.liveSlot { + liveSlot := p.liveSlot.Load() + if p.floor > liveSlot { return p.floor } - return p.liveSlot + return liveSlot } // setForSlotLocked resolves the validator set covering slot. Returns nil (votes @@ -271,45 +288,60 @@ func (p *CertPool) setForSlotLocked(slot uint64) *ValidatorSet { // AddVote ingests one raw (unverified) votor vote. It ONLY shape-checks, bounds // buffering, and parks the vote in its (type, hash) tally. It does NOT touch // dedupe/equivocation/disjointness state — those mutate only after the vote's -// signature verifies (foldTallyLocked). A malformed vote, a vote outside the +// signature verifies (verifyAndFoldTallyWithLockReleased). A malformed vote, a vote outside the // trusted slot window, or a vote past a memory bound is dropped. func (p *CertPool) AddVote(msg VoteMessage) { if msg.Vote.ValidateBasic() != nil || len(msg.Signature) != BLSSignatureSize { return } slot := msg.Vote.Slot + sigKey := sha256.Sum256(msg.Signature) - var emits []Certificate - var verified []VerifiedVote p.mu.Lock() - if slot <= p.floor { - p.snap.VotesRejected++ - p.mu.Unlock() - return - } - // Window anchored to the TRUSTED watermark (floor / replay-observed), not to - // the highest vote slot seen — otherwise an attacker could slide it forward - // vote by vote and retain arbitrarily many future slots. - if anchor := p.windowAnchorLocked(); anchor > 0 && slot > anchor+p.cfg.MaxSlotsAhead { - p.snap.VotesRejected++ - p.mu.Unlock() - return - } - - ps := p.slots[slot] - if ps == nil { - // Hard global bound on retained slots (independent of the window). - if len(p.slots) >= p.cfg.MaxLiveSlots && !p.evictFartherFutureSlotLocked(slot) { + var ps *poolSlot + for { + if slot <= p.floor { + p.snap.VotesRejected++ + p.mu.Unlock() + return + } + // Window anchored to the TRUSTED watermark (floor / replay-observed), not to + // the highest vote slot seen — otherwise an attacker could slide it forward + // vote by vote and retain arbitrarily many future slots. + if anchor := p.windowAnchorLocked(); anchor > 0 && slot > anchor+p.cfg.MaxSlotsAhead { p.snap.VotesRejected++ p.mu.Unlock() return } - ps = &poolSlot{ - tallies: make(map[tallyKey]*tally), - verifiedHash: make(map[voteDedupKey][]solana.Hash), - pendingByRank: make(map[uint16]int), + + ps = p.slots[slot] + if ps == nil { + // Hard global bound on retained slots (independent of the window). + if len(p.slots) >= p.cfg.MaxLiveSlots && !p.evictFartherFutureSlotLocked(slot) { + p.snap.VotesRejected++ + p.mu.Unlock() + return + } + ps = &poolSlot{ + tallies: make(map[tallyKey]*tally), + verifiedHash: make(map[voteDedupKey][]solana.Hash), + pendingByRank: make(map[uint16]int), + } + p.slots[slot] = ps } - p.slots[slot] = ps + // Ordinary arrivals can join the bounded pending maps during BLS work. + // Quota pressure and competing signatures still wait for authentication + // before admission, preserving the first-packet-poisoning protections. + // A queued reward flush also stops new arrivals from extending its drain. + if ps.flushWaiting > 0 || (ps.processing && p.admissionNeedsFoldLocked(ps, msg, sigKey)) { + p.workCond.Wait() + continue + } + break + } + owner := !ps.processing + if owner { + ps.processing = true } tk := tallyKey{Type: msg.Vote.Type, Hash: msg.Vote.BlockHash} @@ -324,7 +356,6 @@ func (p *CertPool) AddVote(msg VoteMessage) { // one. Different block hashes remain separate tallies. forceFold := false if _, done := tl.verified[msg.Rank]; !done { - sigKey := sha256.Sum256(msg.Signature) candidates := tl.pending[msg.Rank] _, duplicate := candidates[sigKey] if !duplicate && ps.pendingByRank[msg.Rank] >= p.cfg.MaxPendingVotesPerRankSlot { @@ -332,7 +363,11 @@ func (p *CertPool) AddVote(msg VoteMessage) { // the rank quota, authenticate that rank's parked candidates and free // the invalid ones before deciding whether the real vote has room. if set := p.setForSlotLocked(slot); set != nil { - verified = append(verified, p.foldPendingRankLocked(slot, ps, msg.Rank, set)...) + p.foldPendingRankLocked(slot, ps, msg.Rank, set) + } + if p.slots[slot] != ps { + p.finishSlotAndUnlock(slot, ps, nil) + return } candidates = tl.pending[msg.Rank] } @@ -342,7 +377,11 @@ func (p *CertPool) AddVote(msg VoteMessage) { atCap := ps.pendingCount >= p.cfg.MaxPendingVotesPerSlot || p.totalPending >= p.cfg.MaxPendingVotesTotal if !rejectIncoming && atCap { if set := p.setForSlotLocked(slot); set != nil { - verified = append(verified, p.foldAllPendingLocked(slot, ps, set)...) + p.foldAllPendingLocked(slot, ps, set) + } + if p.slots[slot] != ps { + p.finishSlotAndUnlock(slot, ps, nil) + return } if p.totalPending >= p.cfg.MaxPendingVotesTotal { p.evictFartherFutureSlotLocked(slot) @@ -382,90 +421,160 @@ func (p *CertPool) AddVote(msg VoteMessage) { if slot > p.highestSlot { p.highestSlot = slot } + if !owner { + ps.dirty = true + p.mu.Unlock() + return + } if forceFold { if set := p.setForSlotLocked(slot); set != nil { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) + } + } + emits := p.drainSlotLocked(slot, ps, false) + p.finishSlotAndUnlock(slot, ps, emits) +} + +func (p *CertPool) admissionNeedsFoldLocked(ps *poolSlot, msg VoteMessage, sigKey [sha256.Size]byte) bool { + tl := ps.tallies[tallyKey{Type: msg.Vote.Type, Hash: msg.Vote.BlockHash}] + if tl != nil { + if _, done := tl.verified[msg.Rank]; done { + return false + } + candidates := tl.pending[msg.Rank] + if _, duplicate := candidates[sigKey]; duplicate { + return false + } + if len(candidates) > 0 { + return true } } - // Keep verified stake fresh for the Votor fallback triggers. Only plain - // notarize/skip arrivals can change the trigger predicates. - if msg.Vote.Type == VoteTypeNotarize || msg.Vote.Type == VoteTypeSkip { - verified = append(verified, p.maybeFoldTriggersLocked(slot, ps)...) + return ps.pendingByRank[msg.Rank] >= p.cfg.MaxPendingVotesPerRankSlot || + ps.pendingCount >= p.cfg.MaxPendingVotesPerSlot || p.totalPending >= p.cfg.MaxPendingVotesTotal +} + +// waitForSlotLocked releases mu while the current owner finishes. Pruning can +// remove or replace the slot, so callers always use the returned current state. +func (p *CertPool) waitForSlotLocked(slot uint64) *poolSlot { + for { + ps := p.slots[slot] + if ps == nil || (!ps.processing && ps.flushWaiting == 0) { + return ps + } + p.workCond.Wait() } - var assembledVerified []VerifiedVote - emits, assembledVerified = p.maybeAssembleLocked(slot, ps) - verified = append(verified, assembledVerified...) - sink := p.verifiedVoteSink - publication := p.reservePublicationLocked(verified) - p.mu.Unlock() +} - p.publishVerifiedVotes(sink, verified, publication) +// drainSlotLocked catches arrivals buffered while the owner was outside mu. +// Sub-threshold votes remain lazy except when preparing a reward footer. +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. +func (p *CertPool) drainSlotLocked(slot uint64, ps *poolSlot, flush bool) []Certificate { + var emits []Certificate + for p.slots[slot] == ps { + ps.dirty = false + if flush { + if set := p.setForSlotLocked(slot); set != nil { + for key, tl := range ps.tallies { + if key.Type == VoteTypeSkip || key.Type == VoteTypeNotarize { + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) + } + } + } + } + p.maybeFoldTriggersLocked(slot, ps) + certs := p.maybeAssembleLocked(slot, ps) + emits = append(emits, certs...) + if !ps.dirty { + break + } + } + return emits +} + +// finishSlotAndUnlock requires p.mu held and returns with p.mu released. +// It takes the publication barrier before making the slot available to a +// flushing caller, then unlocks and emits certificates. Callers must not defer +// an unlock across this call. +func (p *CertPool) finishSlotAndUnlock(slot uint64, ps *poolSlot, emits []Certificate) uint64 { + if p.slots[slot] != ps { + emits = nil + } + target := p.publicationTargetLocked() + ps.processing = false + p.workCond.Broadcast() + p.mu.Unlock() for _, cert := range emits { p.emitCert(cert) } + return target } // OnValidatorSetInstalled retries assembly for buffered slots that resolve to // the newly-installed epoch. Requires a real slot→epoch lookup; without one no // slot can be safely attributed to the epoch, so nothing is retried. func (p *CertPool) OnValidatorSetInstalled(epoch uint64) { - var emits []Certificate - var verified []VerifiedVote p.mu.Lock() + var slots []uint64 if p.epochForSlot != nil { - for slot, ps := range p.slots { - if p.epochForSlot(slot) != epoch { - continue + for slot := range p.slots { + if p.epochForSlot(slot) == epoch { + slots = append(slots, slot) } - verified = append(verified, p.maybeFoldTriggersLocked(slot, ps)...) - certs, newlyVerified := p.maybeAssembleLocked(slot, ps) - emits = append(emits, certs...) - verified = append(verified, newlyVerified...) } } - sink := p.verifiedVoteSink - publication := p.reservePublicationLocked(verified) p.mu.Unlock() - p.publishVerifiedVotes(sink, verified, publication) - for _, cert := range emits { - p.emitCert(cert) + for _, slot := range slots { + p.mu.Lock() + ps := p.waitForSlotLocked(slot) + if ps == nil || p.epochForSlot == nil || p.epochForSlot(slot) != epoch { + p.mu.Unlock() + continue + } + ps.processing = true + emits := p.drainSlotLocked(slot, ps, false) + p.finishSlotAndUnlock(slot, ps, emits) } } // FlushRewardVotes batch-verifies every pending plain skip/notarize vote for a // reward slot. Normal consensus verification stays lazy, but block production // calls this just before building the slot+8 footer so valid below-threshold -// votes are not omitted from reward certificates. +// votes are not omitted from reward certificates. Queued flushes take precedence +// over new slot owners and arrivals; an existing owner finishes first. Pruning +// can invalidate that generation, in which case the current slot is rechecked. func (p *CertPool) FlushRewardVotes(slot uint64) { - var emits []Certificate - var verified []VerifiedVote p.mu.Lock() - ps := p.slots[slot] - set := p.setForSlotLocked(slot) - if ps != nil && set != nil { - for key, tl := range ps.tallies { - if key.Type == VoteTypeSkip || key.Type == VoteTypeNotarize { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) - } + var ps *poolSlot + for { + ps = p.slots[slot] + if ps == nil { + break + } + ps.flushWaiting++ + for p.slots[slot] == ps && ps.processing { + p.workCond.Wait() } - var assembled []VerifiedVote - emits, assembled = p.maybeAssembleLocked(slot, ps) - verified = append(verified, assembled...) + if p.slots[slot] == ps { + break + } + // Pruning or a binding change replaced this generation while waiting. + ps.flushWaiting-- + p.workCond.Broadcast() } - sink := p.verifiedVoteSink - publication := p.reservePublicationLocked(verified) - targetPublication := p.publicationTargetLocked() - p.mu.Unlock() - - p.publishVerifiedVotes(sink, verified, publication) - for _, cert := range emits { - p.emitCert(cert) + if ps == nil { + target := p.publicationTargetLocked() + p.mu.Unlock() + p.waitForPublication(target) + return } - // A vote can finish BLS verification in AddVote just before this flush takes - // the pool lock, then still be in flight to the reward builder. Wait through - // that publication sequence so a footer cannot omit an already-verified vote. - p.waitForPublication(targetPublication) + ps.processing = true + emits := p.drainSlotLocked(slot, ps, true) + ps.flushWaiting-- + target := p.finishSlotAndUnlock(slot, ps, emits) + // Includes publication by an owner that was verifying when flush arrived. + p.waitForPublication(target) } // reservePublicationLocked assigns ordering while p.mu is held. Flush can then @@ -541,6 +650,7 @@ func (p *CertPool) ObserveFloor(finalizedSlot uint64) { delete(p.emitted, key) } } + p.workCond.Broadcast() } p.mu.Unlock() } @@ -557,10 +667,11 @@ func (p *CertPool) Snapshot() CertPoolSnapshot { p.mu.Lock() defer p.mu.Unlock() snap := p.snap + snap.BatchesVerified = p.batchesVerified.Load() snap.Slots = len(p.slots) snap.Floor = p.floor snap.HighestSlot = p.highestSlot - snap.LiveSlot = p.liveSlot + snap.LiveSlot = p.liveSlot.Load() snap.PendingTotal = p.totalPending return snap } @@ -648,10 +759,15 @@ func meets(f Fraction, stake, total uint64) bool { // implementation (Agave) would. This is the ONLY fold policy: one fork-choice // behavior for observer and voting nodes alike; sub-trigger tallies still // cost nothing. -func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) []VerifiedVote { +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. +func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) { + if p.slots[slot] != ps { + return + } set := p.setForSlotLocked(slot) if set == nil { - return nil // votes stay buffered; retried on OnValidatorSetInstalled + return // votes stay buffered; retried on OnValidatorSetInstalled } total := set.TotalStake v := buildTriggerViewLocked(ps, set) @@ -686,22 +802,21 @@ func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) []Verified } if !foldSkip && !foldAllNotar && len(foldNotar) == 0 { - return nil + return } - var verified []VerifiedVote for tk, tl := range ps.tallies { switch tk.Type { case VoteTypeNotarize: if foldAllNotar || foldNotar[tk.Hash] { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } case VoteTypeSkip: if foldSkip { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } } } - return verified + return } // VotorStakes is the verified-stake observation Votor's fallback-trigger @@ -723,12 +838,12 @@ type VotorStakes struct { func (p *CertPool) VerifiedVotorStakes(slot uint64) (VotorStakes, bool) { p.mu.Lock() defer p.mu.Unlock() + ps := p.waitForSlotLocked(slot) set := p.setForSlotLocked(slot) if set == nil { return VotorStakes{}, false } out := VotorStakes{Notarize: make(map[solana.Hash]uint64), TotalStake: set.TotalStake} - ps := p.slots[slot] if ps == nil { return out, true } @@ -804,14 +919,18 @@ func targetsForSlot(ps *poolSlot) []certTarget { // maybeAssembleLocked checks every assemblable target for the slot: folds // pending votes (batch verification) once candidate stake crosses the // threshold, and returns any newly assembled certificates for emission. -func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate, []VerifiedVote) { +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. +func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) []Certificate { + if p.slots[slot] != ps { + return nil + } set := p.setForSlotLocked(slot) if set == nil { - return nil, nil // validator set / epoch not resolvable yet; votes stay buffered + return nil // validator set / epoch not resolvable yet; votes stay buffered } var emits []Certificate - var verified []VerifiedVote for _, target := range targetsForSlot(ps) { key := CertificateKey{Type: target.certType, Slot: slot} if target.certType.HasBlock() { @@ -846,8 +965,11 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate } // Candidate stake crossed: fold pending votes (one pairing per tally). - verified = append(verified, p.foldTallyLocked(slot, ps, base, set)...) - verified = append(verified, p.foldTallyLocked(slot, ps, fb, set)...) + p.verifyAndFoldTallyWithLockReleased(slot, ps, base, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, fb, set) + if p.slots[slot] != ps { + return nil + } verifiedStake := uint64(0) if base != nil { @@ -869,26 +991,39 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate p.snap.CertsEmitted++ emits = append(emits, cert) } - return emits, verified + return emits } -// foldTallyLocked batch-verifies a tally's pending votes. All sign the same +// verifyAndFoldTallyWithLockReleased requires p.mu held on entry and returns +// with p.mu held on every path. The caller must own ps.processing. Verification +// temporarily releases p.mu; retained slot and validator bindings are rechecked +// after reacquiring it before any verified results are installed. +// +// It batch-verifies a tally's pending votes. All sign the same // payload, so randomized weighted pubkey/signature sums need one pairing; // failures bisect to isolate the bad votes (dropped + counted). Only AFTER a // vote verifies does it update the durable per-slot state — the vote-budget / // equivocation ledger (verifiedHash) and base↔fallback disjointness — so raw // votes can never poison those. -func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *ValidatorSet) []VerifiedVote { - if tl == nil || len(tl.pending) == 0 { - return nil +func (p *CertPool) verifyAndFoldTallyWithLockReleased(slot uint64, ps *poolSlot, tl *tally, set *ValidatorSet) { + if p.slots[slot] != ps || tl == nil || len(tl.pending) == 0 { + return } batch := make([]VoteMessage, 0, pendingCandidateCount(tl)) + // Reuse admission keys without hashing each signature again on removal. + var keyStorage [64][sha256.Size]byte + keys := keyStorage[:0] + if cap(batch) > len(keyStorage) { + keys = make([][sha256.Size]byte, 0, cap(batch)) + } for rank, candidates := range tl.pending { count := len(candidates) if int(rank) < len(set.Validators) { - for _, msg := range candidates { + for key, msg := range candidates { batch = append(batch, msg) + keys = append(keys, key) } + continue } else { p.snap.VotesRejected += uint64(count) } @@ -900,10 +1035,50 @@ func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *Va } p.totalPending -= count } + // Keep candidates in the pending maps while verifying. Concurrent arrivals + // can deduplicate against them, and every in-flight byte still consumes the + // normal per-rank, per-slot and global admission budget. + generation := p.epochGeneration + shredVersion := p.verifier.ShredVersion() + p.mu.Unlock() + p.verificationMu.Lock() good := p.verifyBatch(batch, set) + p.verificationMu.Unlock() + p.mu.Lock() + if p.slots[slot] != ps { + return // pruning/eviction already released the pending accounting + } + current := p.setForSlotLocked(slot) + if generation != p.epochGeneration || shredVersion != p.verifier.ShredVersion() || !sameInstalledValidatorSet(current, set) { + // Do not mix an old aggregate with a new epoch/key/stake binding. Retire + // this slot's old state; a later arrival starts afresh with the current set. + p.totalPending -= ps.pendingCount + delete(p.slots, slot) + for key := range p.emitted { + if key.Slot == slot { + delete(p.emitted, key) + } + } + p.workCond.Broadcast() + return + } + for i, msg := range batch { + candidates := tl.pending[msg.Rank] + delete(candidates, keys[i]) + if len(candidates) == 0 { + delete(tl.pending, msg.Rank) + } + ps.pendingCount-- + ps.pendingByRank[msg.Rank]-- + if ps.pendingByRank[msg.Rank] == 0 { + delete(ps.pendingByRank, msg.Rank) + } + p.totalPending-- + } verified := make([]VerifiedVote, 0, len(good)) - for _, msg := range good { + for i := range good { + msg := good[i].message verified = append(verified, VerifiedVote{ Message: msg, Result: VoteVerifyResult{ @@ -951,40 +1126,47 @@ func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *Va } } - pub, err := validatorBLSPubkey(*set, int(msg.Rank)) - if err != nil { - continue - } - var sig bls12381.G2Affine - if _, err := sig.SetBytes(msg.Signature); err != nil { - continue - } - tl.aggPub.Add(&tl.aggPub, &pub) - tl.aggSig.Add(&tl.aggSig, &sig) + // Reuse the exact signature point authenticated by verifyBatch. + tl.aggSig.Add(&tl.aggSig, &good[i].sig) tl.verified[msg.Rank] = struct{}{} tl.stake += set.Validators[msg.Rank].Stake ps.verifiedHash[dk] = append(seen, msg.Vote.BlockHash) } p.snap.BadSignatures += uint64(len(batch) - len(good)) - return verified + // Publish this batch before verifying any newly buffered work. In particular, + // a growing slot must not hold back an already-authenticated quorum. + sink := p.verifiedVoteSink + publication := p.reservePublicationLocked(verified) + p.mu.Unlock() + p.publishVerifiedVotes(sink, verified, publication) + p.mu.Lock() } -func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, set *ValidatorSet) []VerifiedVote { - var verified []VerifiedVote +// Installed sets own immutable parsed-key arrays. Reinstallation, even for the +// same epoch and keys, gets a new array and therefore invalidates in-flight work. +func sameInstalledValidatorSet(a, b *ValidatorSet) bool { + return a != nil && b != nil && a.Epoch == b.Epoch && len(a.parsedPubkeys) > 0 && + len(a.parsedPubkeys) == len(b.parsedPubkeys) && &a.parsedPubkeys[0] == &b.parsedPubkeys[0] +} + +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. +func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, set *ValidatorSet) { for _, tl := range ps.tallies { if len(tl.pending[rank]) != 0 { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } } - return verified + return } -func (p *CertPool) foldAllPendingLocked(slot uint64, ps *poolSlot, set *ValidatorSet) []VerifiedVote { - var verified []VerifiedVote +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. +func (p *CertPool) foldAllPendingLocked(slot uint64, ps *poolSlot, set *ValidatorSet) { for _, tl := range ps.tallies { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } - return verified + return } func (p *CertPool) evictFartherFutureSlotLocked(incoming uint64) bool { @@ -1017,6 +1199,7 @@ func (p *CertPool) evictFartherFutureSlotLocked(incoming uint64) bool { p.totalPending = 0 } delete(p.slots, victim) + p.workCond.Broadcast() return true } @@ -1045,13 +1228,13 @@ type parsedBatchVote struct { // check, bisecting on failure. Unweighted aggregation is insufficient here: // invalid shares can cancel while the downstream pool later keeps only a subset. // Independent random coefficients bind success to every individual member. -func (p *CertPool) verifyBatch(batch []VoteMessage, set *ValidatorSet) []VoteMessage { +func (p *CertPool) verifyBatch(batch []VoteMessage, set *ValidatorSet) []parsedBatchVote { if len(batch) == 0 { return nil } - p.snap.BatchesVerified++ payload, err := EncodeVotePayloadToSign(batch[0].Vote, p.verifier.ShredVersion()) if err != nil { + p.batchesVerified.Add(1) return nil } @@ -1068,31 +1251,80 @@ func (p *CertPool) verifyBatch(batch []VoteMessage, set *ValidatorSet) []VoteMes members = append(members, parsedBatchVote{message: msg, pubkey: pub, sig: sig}) } + return p.verifyParsedBatch(members, payload) +} + +// verifyParsedBatch keeps the owned parsed points through failed-batch +// subdivision and returns only individually bound, verified members. Each +// subdivision still uses fresh random coefficients; parsing is the only work +// reused. Verification does not read or mutate the pool's slot maps. +func (p *CertPool) verifyParsedBatch(members []parsedBatchVote, payload []byte) []parsedBatchVote { + p.batchesVerified.Add(1) if len(members) == 0 { return nil } if len(members) == 1 { if aggregatePairingOK(members[0].pubkey, payload, members[0].sig) { - return []VoteMessage{members[0].message} + return members } return nil } if ok, err := randomizedAggregatePairingOK(members, payload); err == nil && ok { - return batchMessages(members) + return members } else if err != nil { // Entropy failure must reduce performance, never verification strength. return individuallyVerifiedBatch(members, payload) } // Aggregate failed: bisect the structurally-valid subset. - messages := batchMessages(members) - mid := len(messages) / 2 - valid := p.verifyBatch(messages[:mid], set) - valid = append(valid, p.verifyBatch(messages[mid:], set)...) + mid := len(members) / 2 + valid := p.verifyParsedBatch(members[:mid:mid], payload) + valid = append(valid, p.verifyParsedBatch(members[mid:], payload)...) return valid } func randomizedAggregatePairingOK(members []parsedBatchVote, payload []byte) (bool, error) { + // Bucket setup outweighs MultiExp's savings on small batches, including + // the two-candidate collision path and failed-batch subdivisions. Its + // internal task handoffs can also delay verification behind replay when + // only one Go execution thread is available, despite doing less arithmetic. + if len(members) < 16 || runtime.GOMAXPROCS(0) == 1 { + return randomizedAggregatePairingScalarOK(members, payload) + } + return randomizedAggregatePairingMultiExpOK(members, payload) +} + +func randomizedAggregatePairingMultiExpOK(members []parsedBatchVote, payload []byte) (bool, error) { + pubkeys := make([]bls12381.G1Affine, len(members)) + signatures := make([]bls12381.G2Affine, len(members)) + coefficients := make([]blsfr.Element, len(members)) + for i := range members { + coefficient, err := randomNonzeroBatchCoefficient() + if err != nil { + return false, err + } + // Keep independent, nonzero, full-field coefficients and apply the + // same coefficient to each member's public key and signature. + coefficients[i].SetBigInt(coefficient) + pubkeys[i] = members[i].pubkey + signatures[i] = members[i].sig + } + // MultiExp defaults to using all CPUs. Keep its arithmetic concurrency at + // one, and run G1/G2 sequentially, to avoid competing with replay workers. + // Pool-level verification admission remains bounded by verificationMu. + config := ecc.MultiExpConfig{NbTasks: 1} + var aggPub bls12381.G1Affine + if _, err := aggPub.MultiExp(pubkeys, coefficients, config); err != nil { + return false, err + } + var aggSig bls12381.G2Affine + if _, err := aggSig.MultiExp(signatures, coefficients, config); err != nil { + return false, err + } + return aggregatePairingOK(aggPub, payload, aggSig), nil +} + +func randomizedAggregatePairingScalarOK(members []parsedBatchVote, payload []byte) (bool, error) { var aggPub bls12381.G1Affine var aggSig bls12381.G2Affine aggPub.SetInfinity() @@ -1124,24 +1356,16 @@ func randomNonzeroBatchCoefficient() (*big.Int, error) { } } -func individuallyVerifiedBatch(members []parsedBatchVote, payload []byte) []VoteMessage { - valid := make([]VoteMessage, 0, len(members)) +func individuallyVerifiedBatch(members []parsedBatchVote, payload []byte) []parsedBatchVote { + valid := make([]parsedBatchVote, 0, len(members)) for i := range members { if aggregatePairingOK(members[i].pubkey, payload, members[i].sig) { - valid = append(valid, members[i].message) + valid = append(valid, members[i]) } } return valid } -func batchMessages(members []parsedBatchVote) []VoteMessage { - messages := make([]VoteMessage, len(members)) - for i := range members { - messages[i] = members[i].message - } - return messages -} - func aggregatePairingOK(aggPub bls12381.G1Affine, payload []byte, aggSig bls12381.G2Affine) bool { if aggPub.IsInfinity() { return false diff --git a/pkg/alpenglow/certpool_bench_test.go b/pkg/alpenglow/certpool_bench_test.go new file mode 100644 index 000000000..a6864f0af --- /dev/null +++ b/pkg/alpenglow/certpool_bench_test.go @@ -0,0 +1,160 @@ +package alpenglow + +import ( + "crypto/sha256" + "fmt" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" + "github.com/gagliardetto/solana-go" +) + +// Benchmark the full pending-vote fold, including verification, aggregation and +// accounting. Keys and signatures are prepared outside the timed region. +func BenchmarkCertPoolFoldVerifiedBatch(b *testing.B) { + for _, size := range []int{1, 8, 32, 64} { + b.Run(fmt.Sprintf("votes=%d", size), func(b *testing.B) { + verifier, installed, vote, batch := certPoolBenchmarkFixture(b, size) + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return installed.Epoch }) + tl := newTally() + ps := &poolSlot{processing: true, verifiedHash: make(map[voteDedupKey][]solana.Hash), pendingByRank: make(map[uint16]int)} + pool.slots[vote.Slot] = ps + for _, msg := range batch { + tl.pending[msg.Rank] = map[[sha256.Size]byte]VoteMessage{sha256.Sum256(msg.Signature): msg} + ps.pendingByRank[msg.Rank]++ + ps.pendingCount++ + pool.totalPending++ + } + pool.mu.Lock() + pool.verifyAndFoldTallyWithLockReleased(vote.Slot, ps, tl, &installed) + pool.mu.Unlock() + if len(tl.verified) != size || tl.stake != uint64(size) || pool.totalPending != 0 { + b.Fatal("incomplete verified fold") + } + } + }) + } +} + +func certPoolBenchmarkFixture(b testing.TB, size int) (*CertificateVerifier, ValidatorSet, Vote, []VoteMessage) { + stakes := make([]uint64, size) + for i := range stakes { + stakes[i] = 1 + } + set, keys := testBLSValidatorSet(uint64(size), stakes...) + verifier := NewCertificateVerifier() + if err := verifier.SetValidatorSet(set); err != nil { + b.Fatal(err) + } + // Exercise the same cached public-key representation used in production. + installed, ok := verifier.ValidatorSetForEpoch(set.Epoch) + if !ok { + b.Fatal("missing installed validator set") + } + vote := NewSkipVote(500) + payload, err := EncodeVotePayloadToSign(vote, verifier.ShredVersion()) + if err != nil { + b.Fatal(err) + } + point, err := bls12381.HashToG2(payload, []byte(blsHashToPointDST)) + if err != nil { + b.Fatal(err) + } + batch := make([]VoteMessage, size) + for i := range batch { + var sig bls12381.G2Affine + sig.ScalarMultiplication(&point, keys[i]) + raw := sig.RawBytes() + batch[i] = VoteMessage{Vote: vote, Rank: uint16(i), Signature: raw[:]} + } + return verifier, installed, vote, batch +} + +// Observe short pool reads while eight peer-like producers submit one slot. +// The latency metrics, not ns/op (which includes deliberate sampling delays), +// measure whether cryptographic work blocks unrelated pool readers. +func BenchmarkCertPoolConcurrentVotes(b *testing.B) { + verifier, installed, vote, batch := certPoolBenchmarkFixture(b, 64) + waits := make([]int64, 0, 16*b.N) + b.ResetTimer() + for n := 0; n < b.N; n++ { + pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return installed.Epoch }) + pool.NoteLiveSlot(vote.Slot) + var verified atomic.Int64 + pool.SetVerifiedVoteSink(func(VerifiedVote) { verified.Add(1) }) + start := make(chan struct{}) + var producers sync.WaitGroup + for worker := 0; worker < 8; worker++ { + producers.Add(1) + go func(worker int) { + defer producers.Done() + <-start + for i := worker; i < len(batch); i += 8 { + pool.AddVote(batch[i]) + } + }(worker) + } + close(start) + for sample := 0; sample < 16; sample++ { + time.Sleep(100 * time.Microsecond) + t := time.Now() + pool.Snapshot() + waits = append(waits, time.Since(t).Nanoseconds()) + } + producers.Wait() + pool.FlushRewardVotes(vote.Slot) + if verified.Load() != 64 || pool.Snapshot().PendingTotal != 0 { + b.Fatal("lost or duplicated votes") + } + } + b.StopTimer() + slices.Sort(waits) + b.ReportMetric(float64(waits[len(waits)*95/100]), "snapshot-p95-ns") + b.ReportMetric(float64(waits[len(waits)-1]), "snapshot-max-ns") +} + +func BenchmarkCertPoolWeightedPairing(b *testing.B) { + for _, size := range []int{2, 4, 8, 16, 32, 64, 128} { + verifier, set, vote, batch := certPoolBenchmarkFixture(b, size) + payload, err := EncodeVotePayloadToSign(vote, verifier.ShredVersion()) + if err != nil { + b.Fatal(err) + } + members := make([]parsedBatchVote, size) + for i, msg := range batch { + pub, err := validatorBLSPubkey(set, int(msg.Rank)) + if err != nil { + b.Fatal(err) + } + members[i] = parsedBatchVote{message: msg, pubkey: pub} + if _, err := members[i].sig.SetBytes(msg.Signature); err != nil { + b.Fatal(err) + } + } + for _, impl := range []struct { + name string + verify func([]parsedBatchVote, []byte) (bool, error) + }{ + {"Scalar", randomizedAggregatePairingScalarOK}, + {"MultiExp", randomizedAggregatePairingMultiExpOK}, + } { + b.Run(fmt.Sprintf("votes=%d/%s", size, impl.name), func(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + if ok, err := impl.verify(members, payload); err != nil || !ok { + b.Fatalf("verification: %t %v", ok, err) + } + } + }) + } + } +} diff --git a/pkg/alpenglow/certpool_concurrency_test.go b/pkg/alpenglow/certpool_concurrency_test.go new file mode 100644 index 000000000..a97c9eea9 --- /dev/null +++ b/pkg/alpenglow/certpool_concurrency_test.go @@ -0,0 +1,252 @@ +package alpenglow + +import ( + "fmt" + "sync" + "testing" + "time" +) + +func waitCertPoolCall(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("certificate-pool operation did not finish") + } +} + +func certPoolAsync(fn func()) <-chan struct{} { + done := make(chan struct{}) + go func() { defer close(done); fn() }() + return done +} + +// Park a real owner at the production verification gate. No cryptography is +// stubbed; releasing the gate runs the normal randomized verification path. +func parkCertPoolVerification(t *testing.T, pool *CertPool, msg VoteMessage) (func(), <-chan struct{}) { + t.Helper() + pool.verificationMu.Lock() + var once sync.Once + release := func() { once.Do(pool.verificationMu.Unlock) } + t.Cleanup(release) + done := certPoolAsync(func() { pool.AddVote(msg) }) + ready := certPoolAsync(func() { + for pool.Snapshot().PendingTotal == 0 { + time.Sleep(time.Millisecond) + } + }) + waitCertPoolCall(t, ready) // Also proves Snapshot can acquire mu during work. + return release, done +} + +func TestCertPoolOffLockAdmissionAndRewardFlush(t *testing.T) { + pool, set, keys, emitted := newTestPool(t) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + vote := NewSkipVote(500) + first := VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])} + second := VoteMessage{Vote: vote, Rank: 1, Signature: signTestVote(t, vote, keys[1])} + release, owner := parkCertPoolVerification(t, pool, first) + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(second) })) + if got := pool.Snapshot().PendingTotal; got != 2 { + t.Fatalf("pending = %d; in-flight and newly buffered votes must both count", got) + } + flush := certPoolAsync(func() { pool.FlushRewardVotes(vote.Slot) }) + select { + case <-flush: + t.Fatal("flush escaped in-flight verification") + case <-time.After(20 * time.Millisecond): + } + if len(seen) != 0 { + t.Fatal("unverified vote reached sink") + } + release() + waitCertPoolCall(t, owner) + waitCertPoolCall(t, flush) + if len(seen) != 2 || pool.Snapshot().PendingTotal != 0 { + t.Fatalf("flush lost or duplicated work: published=%d snapshot=%+v", len(seen), pool.Snapshot()) + } + if len(*emitted) == 0 { + t.Fatal("buffered arrivals did not drive certificate assembly") + } + for _, cert := range *emitted { + if _, _, err := verifyCertificateWithSet(set, cert, true); err != nil { + t.Fatalf("concurrently assembled certificate failed verification: %v", err) + } + } +} + +func TestCertPoolOffLockPruningDiscardsInFlightResults(t *testing.T) { + pool, _, keys, emitted := newTestPool(t) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + vote := NewSkipVote(500) + release, owner := parkCertPoolVerification(t, pool, VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])}) + waitCertPoolCall(t, certPoolAsync(func() { pool.ObserveFloor(vote.Slot) })) + if got := pool.Snapshot(); got.PendingTotal != 0 || got.Slots != 0 { + t.Fatalf("pruning retained in-flight accounting: %+v", got) + } + release() + waitCertPoolCall(t, owner) + if len(seen) != 0 || len(*emitted) != 0 || pool.Snapshot().PendingTotal != 0 { + t.Fatal("pruned work was published or decremented accounting twice") + } + addVote(t, pool, NewSkipVote(501), 0, keys[0]) + if len(seen) != 1 { + t.Fatal("subsequent live vote was lost") + } +} + +func TestCertPoolOffLockBindingChangeDiscardsResults(t *testing.T) { + for _, kind := range []string{"validator-set", "epoch-lookup"} { + t.Run(kind, func(t *testing.T) { + pool, set, keys, emitted := newTestPool(t) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + vote := NewSkipVote(500) + msg := VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])} + release, owner := parkCertPoolVerification(t, pool, msg) + if kind == "validator-set" { + if err := pool.verifier.SetValidatorSet(set); err != nil { + t.Fatal(err) + } + } else { + pool.SetEpochLookup(func(uint64) uint64 { return set.Epoch }) + } + release() + waitCertPoolCall(t, owner) + if got := pool.Snapshot(); len(seen) != 0 || len(*emitted) != 0 || got.PendingTotal != 0 || got.Slots != 0 { + t.Fatalf("stale binding published or retained state: %+v", got) + } + pool.AddVote(msg) + if len(seen) != 1 { + t.Fatal("vote could not be retried under the current binding") + } + }) + } +} + +func TestCertPoolOffLockPendingBoundAndDuplicates(t *testing.T) { + pool, _, keys, _ := newTestPool(t) + pool.cfg.MaxPendingVotesPerSlot = 2 + pool.cfg.MaxPendingVotesTotal = 2 + vote := NewSkipVote(500) + msgs := make([]VoteMessage, 3) + for i := range msgs { + msgs[i] = VoteMessage{Vote: vote, Rank: uint16(i), Signature: signTestVote(t, vote, keys[i])} + } + release, owner := parkCertPoolVerification(t, pool, msgs[0]) + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(msgs[1]) })) + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(msgs[0]) })) + third := certPoolAsync(func() { pool.AddVote(msgs[2]) }) + select { + case <-third: + t.Fatal("capacity-pressure admission did not wait for authentication") + case <-time.After(20 * time.Millisecond): + } + if got := pool.Snapshot().PendingTotal; got != 2 { + t.Fatalf("in-flight votes escaped bounds or duplicate consumed capacity: %d", got) + } + release() + waitCertPoolCall(t, owner) + waitCertPoolCall(t, third) + pool.FlushRewardVotes(vote.Slot) + if got := pool.Snapshot(); got.PendingTotal != 0 || got.VotesAccepted != 3 { + t.Fatalf("capacity wakeup lost or duplicated votes: %+v", got) + } +} + +func TestCertPoolOffLockEvictionDoesNotResurrectSlot(t *testing.T) { + set, keys := testBLSValidatorSet(100, 40, 30, 15, 10, 5) + verifier := NewCertificateVerifier() + if err := verifier.SetValidatorSet(set); err != nil { + t.Fatal(err) + } + pool := NewCertPool(CertPoolConfig{MaxLiveSlots: 1}, verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return set.Epoch }) + pool.NoteLiveSlot(100) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + future := NewSkipVote(110) + release, owner := parkCertPoolVerification(t, pool, VoteMessage{Vote: future, Rank: 0, Signature: signTestVote(t, future, keys[0])}) + near := NewSkipVote(101) + msg := VoteMessage{Vote: near, Rank: 4, Signature: signTestVote(t, near, keys[4])} + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(msg) })) + if got := pool.Snapshot(); got.PendingTotal != 1 || got.Slots != 1 { + t.Fatalf("eviction accounting mismatch: %+v", got) + } + release() + waitCertPoolCall(t, owner) + pool.FlushRewardVotes(near.Slot) + if len(seen) != 1 || (<-seen).Message.Vote.Slot != near.Slot || pool.Snapshot().PendingTotal != 0 { + t.Fatal("evicted work displaced or corrupted the nearer slot") + } +} + +func TestCertPoolRewardFlushPriority(t *testing.T) { + for _, prune := range []bool{false, true} { + t.Run(fmt.Sprintf("prune=%t", prune), func(t *testing.T) { + pool, _, keys, _ := newTestPool(t) + vote := NewSkipVote(500) + // A below-threshold vote must be published by the reward flush. + pool.AddVote(VoteMessage{Vote: vote, Rank: 4, Signature: signTestVote(t, vote, keys[4])}) + pool.mu.Lock() + ps := pool.slots[vote.Slot] + ps.processing = true // Hold ownership until both flushes are queued. + pool.mu.Unlock() + flush1 := certPoolAsync(func() { pool.FlushRewardVotes(vote.Slot) }) + flush2 := certPoolAsync(func() { pool.FlushRewardVotes(vote.Slot) }) + deadline := time.Now().Add(2 * time.Second) + for { + pool.mu.Lock() + waiting := ps.flushWaiting + pool.mu.Unlock() + if waiting == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("flushes did not register") + } + time.Sleep(time.Millisecond) + } + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + t.Cleanup(func() { once.Do(func() { close(release) }) }) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { + if v.Message.Rank == 4 { + close(entered) + <-release + } + }) + msg := VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])} + arrival := certPoolAsync(func() { pool.AddVote(msg) }) + if prune { + pool.ObserveFloor(vote.Slot) + } else { + pool.mu.Lock() + ps.processing = false + pool.workCond.Broadcast() + pool.mu.Unlock() + waitCertPoolCall(t, entered) + if got := pool.Snapshot().VotesAccepted; got != 1 { + t.Fatalf("new arrival overtook reward flush: accepted=%d", got) + } + select { + case <-arrival: + t.Fatal("arrival escaped active flush") + default: + } + } + once.Do(func() { close(release) }) + waitCertPoolCall(t, flush1) + waitCertPoolCall(t, flush2) + waitCertPoolCall(t, arrival) + pool.mu.Lock() + defer pool.mu.Unlock() + if ps.flushWaiting != 0 { + t.Fatalf("leaked flush waiters: %d", ps.flushWaiting) + } + }) + } +} diff --git a/pkg/alpenglow/certpool_points_test.go b/pkg/alpenglow/certpool_points_test.go new file mode 100644 index 000000000..4f76ad01e --- /dev/null +++ b/pkg/alpenglow/certpool_points_test.go @@ -0,0 +1,102 @@ +package alpenglow + +import ( + "bytes" + "testing" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" +) + +func TestCertPoolVerifiedPointsMatchIndividualVerification(t *testing.T) { + for _, mixed := range []bool{false, true} { + name := "valid" + if mixed { + name = "mixed-invalid" + } + t.Run(name, func(t *testing.T) { + pool, set, keys, _ := newTestPool(t) + vote := NewSkipVote(500) + var batch []VoteMessage + for i, key := range keys { + signedVote := vote + if mixed && i%2 == 1 { + signedVote = NewSkipVote(501) // Valid point, wrong payload, in both halves. + } + batch = append(batch, VoteMessage{Vote: vote, Rank: uint16(i), Signature: signTestVote(t, signedVote, key)}) + } + if mixed { + var infinity bls12381.G2Affine + infinity.SetInfinity() + raw := infinity.RawBytes() + batch = append(batch, + VoteMessage{Vote: vote, Rank: 0, Signature: []byte{0xff}}, + VoteMessage{Vote: vote, Rank: 1, Signature: raw[:]}, + VoteMessage{Vote: vote, Rank: uint16(len(keys)), Signature: batch[0].Signature}, + ) + } + var expected []VoteMessage + for _, msg := range batch { + if _, err := verifyVoteMessageWithSet(set, msg); err == nil { + expected = append(expected, msg) + } + } + verified := pool.verifyBatch(batch, &set) + if len(verified) != len(expected) { + t.Fatalf("verified %d members, want %d", len(verified), len(expected)) + } + for i, member := range verified { + if member.message.Rank != expected[i].Rank || member.message.Vote != expected[i].Vote || !bytes.Equal(member.message.Signature, expected[i].Signature) { + t.Fatalf("member %d does not match its individually verified message", i) + } + raw := member.sig.RawBytes() + if !bytes.Equal(raw[:], expected[i].Signature) { + t.Fatalf("member %d retained the wrong signature point", i) + } + pub, err := validatorBLSPubkey(set, int(expected[i].Rank)) + if err != nil || !member.pubkey.Equal(&pub) { + t.Fatalf("member %d retained the wrong public key", i) + } + } + if mixed && pool.Snapshot().BatchesVerified <= 1 { + t.Fatal("mixed batch did not exercise recursive verification") + } + }) + } +} + +// Entropy failure uses this individual-verification path instead of accepting +// an unweighted aggregate. Reusing points must preserve its filtering too. +func TestIndividuallyVerifiedBatchRetainsOnlyValidPoints(t *testing.T) { + _, set, keys, _ := newTestPool(t) + vote := NewSkipVote(502) + payload, err := EncodeVotePayloadToSign(vote, 0) + if err != nil { + t.Fatal(err) + } + var members []parsedBatchVote + for i, key := range keys { + signedVote := vote + if i%2 == 1 { + signedVote = NewSkipVote(503) + } + msg := VoteMessage{Vote: vote, Rank: uint16(i), Signature: signTestVote(t, signedVote, key)} + pub, err := validatorBLSPubkey(set, i) + if err != nil { + t.Fatal(err) + } + var sig bls12381.G2Affine + if _, err := sig.SetBytes(msg.Signature); err != nil { + t.Fatal(err) + } + members = append(members, parsedBatchVote{message: msg, pubkey: pub, sig: sig}) + } + verified := individuallyVerifiedBatch(members, payload) + if len(verified) != 3 { + t.Fatalf("verified %d, want 3", len(verified)) + } + for i, member := range verified { + if member.message.Rank != uint16(i*2) || !member.sig.Equal(&members[i*2].sig) || !member.pubkey.Equal(&members[i*2].pubkey) { + t.Fatalf("wrong verified member at %d", i) + } + } +} diff --git a/pkg/alpenglow/certpool_progress_test.go b/pkg/alpenglow/certpool_progress_test.go new file mode 100644 index 000000000..6732dce4b --- /dev/null +++ b/pkg/alpenglow/certpool_progress_test.go @@ -0,0 +1,112 @@ +package alpenglow + +import ( + "sync" + "testing" + "time" +) + +func TestCertPoolProgressDoesNotWaitForVerificationLock(t *testing.T) { + pool := NewCertPool(DefaultCertPoolConfig(), NewCertificateVerifier(), nil) + pool.mu.Lock() // The same lock held while verifying incoming BLS votes. + done := make(chan struct{}) + go func() { + pool.NoteLiveSlot(123) + close(done) + }() + completed := false + select { + case <-done: + completed = true + case <-time.After(time.Second): + } + pool.mu.Unlock() + <-done + if !completed { + t.Fatal("trusted replay progress waited for the verification lock") + } + if got := pool.Snapshot().LiveSlot; got != 123 { + t.Fatalf("live slot = %d, want 123", got) + } +} + +func TestCertPoolConcurrentProgressRemainsMonotonic(t *testing.T) { + pool := NewCertPool(DefaultCertPoolConfig(), NewCertificateVerifier(), nil) + const workers, updates = 16, 256 + start := make(chan struct{}) + var wg sync.WaitGroup + for worker := range workers { + wg.Go(func() { + <-start + for n := range updates { + slot := uint64(n*workers + worker + 1) + pool.NoteLiveSlot(slot) + pool.NoteLiveSlot(slot / 2) // Stale updates race newer progress. + } + }) + } + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + close(start) + var previous uint64 + for { + live := pool.Snapshot().LiveSlot + pool.mu.Lock() + anchor := pool.windowAnchorLocked() + pool.mu.Unlock() + if live < previous || anchor < live { + t.Errorf("progress regressed: previous=%d snapshot=%d anchor=%d", previous, live, anchor) + } + previous = anchor + select { + case <-finished: + pool.NoteLiveSlot(0) + pool.NoteLiveSlot(1) + if got := pool.Snapshot().LiveSlot; got != workers*updates { + t.Fatalf("live slot = %d, want %d", got, workers*updates) + } + return + default: + } + } +} + +func TestCertPoolProgressPreservesTrustedVoteWindow(t *testing.T) { + set, keys := testBLSValidatorSet(100, 40, 30, 15, 10, 5) + verifier := NewCertificateVerifier() + if err := verifier.SetValidatorSet(set); err != nil { + t.Fatal(err) + } + pool := NewCertPool(CertPoolConfig{MaxSlotsAhead: 10}, verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return set.Epoch }) + pool.NoteLiveSlot(100) + checkVote := func(slot uint64, rejected bool, live uint64) { + t.Helper() + before := pool.Snapshot() + addVote(t, pool, NewSkipVote(slot), 4, keys[4]) + after := pool.Snapshot() + wantRejected := before.VotesRejected + if rejected { + wantRejected++ + } + if after.VotesRejected != wantRejected { + t.Fatalf("slot %d: rejected=%d, want %d", slot, after.VotesRejected, wantRejected) + } + if after.LiveSlot != live { + t.Fatalf("raw vote at %d moved trusted progress to %d, want %d", slot, after.LiveSlot, live) + } + } + checkVote(110, false, 100) // Inclusive upper edge. + checkVote(111, true, 100) // Accepted raw votes cannot slide the window. + pool.NoteLiveSlot(90) + checkVote(111, true, 100) + pool.NoteLiveSlot(101) + checkVote(111, false, 101) + pool.ObserveFloor(120) + checkVote(120, true, 101) // Finalized floor still rejects old votes. + checkVote(130, false, 101) // Floor can anchor the window above replay. + checkVote(131, true, 101) +} diff --git a/pkg/alpenglow/certpool_test.go b/pkg/alpenglow/certpool_test.go index 8c7c36789..24970345c 100644 --- a/pkg/alpenglow/certpool_test.go +++ b/pkg/alpenglow/certpool_test.go @@ -1,6 +1,7 @@ package alpenglow import ( + "fmt" "math/big" "testing" "time" @@ -805,3 +806,50 @@ func TestCertPoolEmitsOnce(t *testing.T) { t.Fatalf("no new certs expected, went from %d to %d", n, len(*emitted)) } } + +// Exercise both aggregation paths, including a malformed member at every +// position, a different signed payload, and failed-batch subdivision. Valid +// votes must survive independently of which member causes the batch to fail. +func TestCertPoolBatchAggregationPaths(t *testing.T) { + for _, size := range []int{2, 8, 16, 64} { + t.Run(fmt.Sprintf("votes=%d", size), func(t *testing.T) { + verifier, set, vote, batch := certPoolBenchmarkFixture(t, size) + pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) + payload, err := EncodeVotePayloadToSign(vote, verifier.ShredVersion()) + if err != nil { + t.Fatal(err) + } + members := pool.verifyBatch(batch, &set) + if len(members) != size { + t.Fatal("valid batch rejected") + } + var tweak bls12381.G2Affine + tweak.ScalarMultiplicationBase(big.NewInt(1234567)) + for i := range members { + bad := append([]parsedBatchVote(nil), members...) + bad[i].sig.Add(&bad[i].sig, &tweak) + if ok, err := randomizedAggregatePairingOK(bad, payload); err != nil || ok { + t.Fatalf("bad member %d: accepted=%t err=%v", i, ok, err) + } + } + wrongPayload := append([]byte(nil), payload...) + wrongPayload[0] ^= 1 + if ok, err := randomizedAggregatePairingOK(members, wrongPayload); err != nil || ok { + t.Fatalf("wrong payload: accepted=%t err=%v", ok, err) + } + // Preserve the unweighted sum while corrupting two shares in a + // large batch; subdivision must keep exactly the honest members. + members[0].sig.Add(&members[0].sig, &tweak) + members[len(members)-1].sig.Sub(&members[len(members)-1].sig, &tweak) + valid := pool.verifyParsedBatch(members, payload) + if len(valid) != size-2 { + t.Fatalf("verified %d, want %d", len(valid), size-2) + } + for _, member := range valid { + if member.message.Rank == 0 || int(member.message.Rank) == size-1 { + t.Fatal("invalid share survived") + } + } + }) + } +} diff --git a/pkg/alpenglow/observer.go b/pkg/alpenglow/observer.go index 9b84e58cb..8f3470bec 100644 --- a/pkg/alpenglow/observer.go +++ b/pkg/alpenglow/observer.go @@ -95,6 +95,17 @@ type Observer struct { replayBlocks map[uint64]BlockID replayOrder []uint64 replayChecks map[CertificateKey]certificateReplayCheck + // Only retained, block-bearing certificates that have not yet been checked + // against replay belong here. Checked history stays in certificates and + // replayChecks for diagnostics/deduplication, but need not be scanned on + // every block (including skipped slots). This is an in-memory observer + // index, not voting authorization or durable crash-recovery state. + pendingReplayCertificates map[CertificateKey]BlockID + // Votes do not change certificate/replay reconciliation. Reuse its exact + // statistics until one of those inputs changes instead of scanning every + // retained certificate for each incoming vote. + pendingStats certificateReplayPendingStats + pendingStatsValid bool votesObserved uint64 certificatesObserved uint64 @@ -149,11 +160,12 @@ func NewObserverWithConfig(cfg ObserverConfig) *Observer { cfg.MaxTrackedReplayBlocks = DefaultMaxTrackedReplayBlocks } return &Observer{ - cfg: cfg, - votes: make(map[VoteMessageKey]VoteMessage), - certificates: make(map[CertificateKey]Certificate), - replayBlocks: make(map[uint64]BlockID), - replayChecks: make(map[CertificateKey]certificateReplayCheck), + cfg: cfg, + votes: make(map[VoteMessageKey]VoteMessage), + certificates: make(map[CertificateKey]Certificate), + replayBlocks: make(map[uint64]BlockID), + replayChecks: make(map[CertificateKey]certificateReplayCheck), + pendingReplayCertificates: make(map[CertificateKey]BlockID), } } @@ -211,6 +223,7 @@ func (o *Observer) ObserveCertificate(cert Certificate) (Observation, error) { key := cert.Key() _, exists := o.certificates[key] if !exists { + o.pendingStatsValid = false tracked := o.trackCertificateLocked(key, cert) o.certificatesObserved++ o.applyCertificateLocked(cert) @@ -225,6 +238,7 @@ func (o *Observer) ObserveCertificate(cert Certificate) (Observation, error) { func (o *Observer) ObserveReplayBlock(obs ReplayBlockObservation) Observation { o.mu.Lock() defer o.mu.Unlock() + o.pendingStatsValid = false if obs.At.IsZero() { obs.At = time.Now() @@ -260,8 +274,8 @@ func (o *Observer) ObserveReplayResult(obs ReplayResultObservation) Observation } func (o *Observer) Snapshot() Snapshot { - o.mu.RLock() - defer o.mu.RUnlock() + o.mu.Lock() + defer o.mu.Unlock() return o.snapshotLocked() } @@ -339,12 +353,16 @@ func (o *Observer) trackCertificateLocked(key CertificateKey, cert Certificate) return false } o.certificates[key] = cert + if block, ok := cert.Block(); ok && block.HasHash() { + o.pendingReplayCertificates[key] = block + } o.certOrder = append(o.certOrder, key) for len(o.certificates) > o.cfg.MaxTrackedCertificates { old := o.certOrder[0] o.certOrder = o.certOrder[1:] delete(o.certificates, old) delete(o.replayChecks, old) + delete(o.pendingReplayCertificates, old) } return true } @@ -369,12 +387,10 @@ func (o *Observer) checkReplayBlockCertificatesLocked(block BlockID) { if !block.HasHash() { return } - for key, cert := range o.certificates { - certBlock, ok := cert.Block() - if !ok || certBlock.Slot != block.Slot { - continue + for key, certBlock := range o.pendingReplayCertificates { + if certBlock.Slot == block.Slot { + o.checkCertificateReplayLocked(key, o.certificates[key]) } - o.checkCertificateReplayLocked(key, cert) } } @@ -410,6 +426,7 @@ func (o *Observer) checkCertificateReplayLocked(key CertificateKey, cert Certifi } } o.replayChecks[key] = check + delete(o.pendingReplayCertificates, key) } type certificateReplayPendingStats struct { @@ -423,30 +440,24 @@ type certificateReplayPendingStats struct { func (o *Observer) certificateReplayPendingStatsLocked() certificateReplayPendingStats { var stats certificateReplayPendingStats - for key, cert := range o.certificates { - if _, checked := o.replayChecks[key]; checked { - continue + for _, certBlock := range o.pendingReplayCertificates { + stats.count++ + if stats.oldestSlot == 0 || certBlock.Slot < stats.oldestSlot { + stats.oldestSlot = certBlock.Slot } - certBlock, ok := cert.Block() - if ok && certBlock.HasHash() { - stats.count++ - if stats.oldestSlot == 0 || certBlock.Slot < stats.oldestSlot { - stats.oldestSlot = certBlock.Slot - } - if certBlock.Slot > stats.newestSlot { - stats.newestSlot = certBlock.Slot - } - if o.oldestReplayBlockSlot != 0 && certBlock.Slot < o.oldestReplayBlockSlot { - stats.preWindow++ - } - if o.oldestReplayBlockSlot != 0 && - o.latestReplayBlockSlot != 0 && - certBlock.Slot >= o.oldestReplayBlockSlot && - certBlock.Slot <= o.latestReplayBlockSlot { - stats.mature++ - if stats.matureOldestSlot == 0 || certBlock.Slot < stats.matureOldestSlot { - stats.matureOldestSlot = certBlock.Slot - } + if certBlock.Slot > stats.newestSlot { + stats.newestSlot = certBlock.Slot + } + if o.oldestReplayBlockSlot != 0 && certBlock.Slot < o.oldestReplayBlockSlot { + stats.preWindow++ + } + if o.oldestReplayBlockSlot != 0 && + o.latestReplayBlockSlot != 0 && + certBlock.Slot >= o.oldestReplayBlockSlot && + certBlock.Slot <= o.latestReplayBlockSlot { + stats.mature++ + if stats.matureOldestSlot == 0 || certBlock.Slot < stats.matureOldestSlot { + stats.matureOldestSlot = certBlock.Slot } } } @@ -454,7 +465,11 @@ func (o *Observer) certificateReplayPendingStatsLocked() certificateReplayPendin } func (o *Observer) snapshotLocked() Snapshot { - pending := o.certificateReplayPendingStatsLocked() + if !o.pendingStatsValid { + o.pendingStats = o.certificateReplayPendingStatsLocked() + o.pendingStatsValid = true + } + pending := o.pendingStats return Snapshot{ VotesObserved: o.votesObserved, CertificatesObserved: o.certificatesObserved, diff --git a/pkg/alpenglow/observer_bench_test.go b/pkg/alpenglow/observer_bench_test.go new file mode 100644 index 000000000..a792015cd --- /dev/null +++ b/pkg/alpenglow/observer_bench_test.go @@ -0,0 +1,57 @@ +package alpenglow + +import ( + "fmt" + "testing" +) + +func BenchmarkObserverVoteWithRetainedCertificates(b *testing.B) { + o := NewObserver() + for slot := uint64(1); slot <= DefaultMaxTrackedCertificates; slot++ { + _, err := o.ObserveCertificate(Certificate{Type: CertificateNotarize, Slot: slot, + BlockHash: testHash(1), IncludedStake: 80, TotalStake: 100}) + if err != nil { + b.Fatal(err) + } + } + msg := VoteMessage{Vote: NewSkipVote(5000), Rank: 1} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := o.ObserveVote(msg); err != nil { + b.Fatal(err) + } + } +} + +// Model a full diagnostic history, with a small unresolved frontier or an +// entirely unresolved history as a worst case. These are observer-only costs: +// no transactions, BLS verification, or network traffic are included. +func BenchmarkObserverEmptyReplay(b *testing.B) { + for _, pending := range []int{0, 32, DefaultMaxTrackedCertificates} { + for _, skips := range []int{0, 4} { + b.Run(fmt.Sprintf("pending=%d/skips=%d", pending, skips), func(b *testing.B) { + o := NewObserver() + for slot := uint64(1); slot <= DefaultMaxTrackedCertificates; slot++ { + if slot <= uint64(DefaultMaxTrackedCertificates-pending) { + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: slot, Hash: testHash(1)}}) + } + _, err := o.ObserveCertificate(Certificate{Type: CertificateNotarize, Slot: slot, + BlockHash: testHash(1), IncludedStake: 80, TotalStake: 100}) + if err != nil { + b.Fatal(err) + } + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + base := uint64(DefaultMaxTrackedCertificates + 1 + i*(skips+1)) + for n := 0; n < skips; n++ { + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: base + uint64(n)}}) + } + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: base + uint64(skips), Hash: testHash(1)}}) + } + }) + } + } +} diff --git a/pkg/alpenglow/observer_stats_test.go b/pkg/alpenglow/observer_stats_test.go new file mode 100644 index 000000000..bf81999de --- /dev/null +++ b/pkg/alpenglow/observer_stats_test.go @@ -0,0 +1,127 @@ +package alpenglow + +import ( + "fmt" + "math/rand" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestObserverPendingStatsMatchFullScan(t *testing.T) { + // Exercise eviction, duplicate certificates, out-of-order and hashless + // replay, and retained matches/mismatches with and without retention. + for _, retention := range []int{0, 1, 8, 64} { + t.Run(fmt.Sprint(retention), func(t *testing.T) { + o := NewObserverWithConfig(ObserverConfig{MaxTrackedVotes: 8, + MaxTrackedCertificates: retention, MaxTrackedReplayBlocks: retention}) + rng := rand.New(rand.NewSource(37)) + for i := 0; i < 1000; i++ { + slot := uint64(rng.Intn(80) + 1) + switch rng.Intn(5) { + case 0, 1: + typ := CertificateNotarize + if i%3 == 0 { + typ = CertificateFinalizeFast + } + _, err := o.ObserveCertificate(Certificate{Type: typ, Slot: slot, + BlockHash: testHash(byte(slot%3 + 1)), IncludedStake: 80, TotalStake: 100}) + require.NoError(t, err) + case 2: + block := BlockID{Slot: slot} + if i%5 != 0 { + block.Hash = testHash(byte(slot%4 + 1)) + } + o.ObserveReplayBlock(ReplayBlockObservation{Block: block}) + case 3: + _, err := o.ObserveVote(VoteMessage{Vote: NewSkipVote(slot), Rank: 1}) + require.NoError(t, err) + case 4: + o.ObserveReplayResult(ReplayResultObservation{Slot: slot}) + } + o.Snapshot() + o.mu.RLock() + cached, scanned := o.pendingStats, observerPendingStatsFullScan(o) + pending := make(map[CertificateKey]BlockID) + for key, cert := range o.certificates { + if _, checked := o.replayChecks[key]; !checked { + if block, ok := cert.Block(); ok && block.HasHash() { + pending[key] = block + } + } + } + require.Equal(t, pending, o.pendingReplayCertificates, "operation %d", i) + o.mu.RUnlock() + require.Equal(t, scanned, cached, "operation %d", i) + } + }) + } +} + +func TestObserverConcurrentSnapshotsAndReconciliation(t *testing.T) { + o := NewObserver() + var wg sync.WaitGroup + for worker := 0; worker < 4; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + for slot := uint64(1); slot <= 100; slot++ { + switch worker { + case 0: + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: slot, Hash: testHash(1)}}) + case 1: + _, err := o.ObserveCertificate(Certificate{Type: CertificateNotarize, Slot: slot, + BlockHash: testHash(1), IncludedStake: 80, TotalStake: 100}) + if err != nil { + t.Error(err) + } + case 2: + _, err := o.ObserveVote(VoteMessage{Vote: NewSkipVote(slot), Rank: 1}) + if err != nil { + t.Error(err) + } + case 3: + o.Snapshot() + } + } + }(worker) + } + wg.Wait() + snapshot := o.Snapshot() + require.Equal(t, uint64(100), snapshot.CertificateReplayMatches) + require.Zero(t, snapshot.CertificateReplayPending) +} + +// Independent reference retains the original scan over every retained certificate. +func observerPendingStatsFullScan(o *Observer) certificateReplayPendingStats { + var stats certificateReplayPendingStats + for key, cert := range o.certificates { + if _, checked := o.replayChecks[key]; checked { + continue + } + certBlock, ok := cert.Block() + if ok && certBlock.HasHash() { + stats.count++ + if stats.oldestSlot == 0 || certBlock.Slot < stats.oldestSlot { + stats.oldestSlot = certBlock.Slot + } + if certBlock.Slot > stats.newestSlot { + stats.newestSlot = certBlock.Slot + } + if o.oldestReplayBlockSlot != 0 && certBlock.Slot < o.oldestReplayBlockSlot { + stats.preWindow++ + } + if o.oldestReplayBlockSlot != 0 && + o.latestReplayBlockSlot != 0 && + certBlock.Slot >= o.oldestReplayBlockSlot && + certBlock.Slot <= o.latestReplayBlockSlot { + stats.mature++ + if stats.matureOldestSlot == 0 || certBlock.Slot < stats.matureOldestSlot { + stats.matureOldestSlot = certBlock.Slot + } + } + } + } + return stats +} diff --git a/pkg/alpenglow/peer_sender.go b/pkg/alpenglow/peer_sender.go new file mode 100644 index 000000000..73c9176f6 --- /dev/null +++ b/pkg/alpenglow/peer_sender.go @@ -0,0 +1,181 @@ +package alpenglow + +import ( + "errors" + "sync" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/quic-go/quic-go" +) + +const ( + defaultVotorPeerSendQueue = 256 + votorSendTimeout = time.Second + votorSendWatchInterval = 100 * time.Millisecond +) + +// VotorPeerQueueStats describes local queueing, not remote delivery. Counters +// here cover the current connection; broadcaster totals survive reconnects. +type VotorPeerQueueStats struct { + Identity solana.PublicKey `json:"identity"` + Address string `json:"address"` + Queued int `json:"queued"` + SendingFor time.Duration `json:"sending_for_ns"` + LastQueueDelay time.Duration `json:"last_queue_delay_ns"` + MaxQueueDelay time.Duration `json:"max_queue_delay_ns"` + QueueDrops uint64 `json:"queue_drops"` +} + +type votorDatagram struct { + payload []byte // immutable, shared across the peer queues + queuedAt time.Time +} + +// Each authenticated connection owns one sender. No mutex is held while +// SendDatagram blocks on quic-go's bounded datagram queue. +type votorPeerSender struct { + b *VotorBroadcaster + peer VotorPeer + conn *quic.Conn + queue chan votorDatagram + done chan struct{} + + mu sync.Mutex + closed bool + sendingSince time.Time + lastQueueDelay time.Duration + maxQueueDelay time.Duration + queueDrops uint64 +} + +func (s *votorPeerSender) enqueue(job votorDatagram) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed || s.conn.Context().Err() != nil { + s.b.sendsSkipped.Add(1) + return + } + select { + case s.queue <- job: + default: + // A full queue rejects only this peer's copy. As before, fanout is + // best effort; the global Enqueue error contract is unchanged. + s.queueDrops++ + s.b.peerQueueDrops.Add(1) + s.b.dropped.Add(1) + } +} + +func (s *votorPeerSender) run() { + defer s.b.wg.Done() + defer close(s.done) + // Cover both remote-close select and close detected after dequeuing a job. + // The queue is bounded/deduplicated; shutdown, departure and a healthy + // replacement connection suppress obsolete reconnect requests. + defer s.b.queueConnect(s.peer.Identity) + defer func() { + s.mu.Lock() + s.closed = true + // Old-connection work is not replayed onto a new address/connection. + // Account for every queued copy discarded on failure or shutdown. + for { + select { + case <-s.queue: + s.b.peerQueueDiscarded.Add(1) + default: + s.mu.Unlock() + return + } + } + }() + for { + select { + case <-s.b.done: + return + case <-s.conn.Context().Done(): + return + case job := <-s.queue: + s.mu.Lock() + if s.closed || s.conn.Context().Err() != nil || s.b.closed.Load() { + s.mu.Unlock() + s.b.peerQueueDiscarded.Add(1) + return + } + now := time.Now() + s.sendingSince = now + s.lastQueueDelay = now.Sub(job.queuedAt) + s.maxQueueDelay = max(s.maxQueueDelay, s.lastQueueDelay) + for old := s.b.peerQueueMaxDelay.Load(); int64(s.lastQueueDelay) > old; old = s.b.peerQueueMaxDelay.Load() { + if s.b.peerQueueMaxDelay.CompareAndSwap(old, int64(s.lastQueueDelay)) { + break + } + } + if s.lastQueueDelay >= votorSendTimeout { + // Do not feed an already-stale backlog into a briefly writable + // QUIC queue between watchdog ticks. Retire this connection. + s.closed = true + s.mu.Unlock() + s.b.peerQueueDiscarded.Add(1) + s.timeout() + return + } + s.mu.Unlock() + err := s.conn.SendDatagram(job.payload) + s.mu.Lock() + s.sendingSince = time.Time{} + s.mu.Unlock() + if err != nil { + s.b.recordSendError(s.peer, err) + var tooLarge *quic.DatagramTooLargeError + if errors.As(err, &tooLarge) { + continue + } + s.b.dropConnection(s.peer.Identity, s.conn) + s.b.queueConnect(s.peer.Identity) + return + } + s.b.sends.Add(1) + } + } +} + +func (s *votorPeerSender) stats() VotorPeerQueueStats { + s.mu.Lock() + defer s.mu.Unlock() + var sendingFor time.Duration + if !s.sendingSince.IsZero() { + sendingFor = time.Since(s.sendingSince) + } + return VotorPeerQueueStats{ + Identity: s.peer.Identity, Address: s.peer.Addr.String(), + Queued: len(s.queue), SendingFor: sendingFor, + LastQueueDelay: s.lastQueueDelay, MaxQueueDelay: s.maxQueueDelay, QueueDrops: s.queueDrops, + } +} + +func (b *VotorBroadcaster) expirePeerSends(now time.Time) { + senders, _ := b.connectedSenders() + for _, s := range senders { + s.mu.Lock() + expired := !s.closed && !s.sendingSince.IsZero() && s.lastQueueDelay+now.Sub(s.sendingSince) >= votorSendTimeout + if expired { + // Serialize with send completion so a late watchdog cannot close a + // later, unrelated send after the blocked operation has finished. + s.closed = true + } + s.mu.Unlock() + if expired { + s.timeout() + } + } +} + +// Caller must first claim the timeout by setting closed under s.mu. This makes +// the dequeue check and watchdog mutually exclusive and counts one timeout. +func (s *votorPeerSender) timeout() { + s.b.peerSendTimeouts.Add(1) + // Closing wakes SendDatagram without leaking a timeout goroutine. + s.b.dropConnection(s.peer.Identity, s.conn) + s.b.queueConnect(s.peer.Identity) +} diff --git a/pkg/alpenglow/peer_sender_recovery_test.go b/pkg/alpenglow/peer_sender_recovery_test.go new file mode 100644 index 000000000..3f88446ea --- /dev/null +++ b/pkg/alpenglow/peer_sender_recovery_test.go @@ -0,0 +1,136 @@ +package alpenglow + +import ( + "context" + "crypto/ed25519" + "crypto/tls" + "net" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Real authenticated connection with no reconciliation loop or connection +// workers. Any reconnect job must come from the sender, never the periodic tick. +func passiveVotorSender(t *testing.T) (*VotorBroadcaster, *votorPeerSender, *Receiver) { + t.Helper() + serverIdentity := ed25519.NewKeyFromSeed(bytesOf(181, ed25519.SeedSize)) + r, err := NewReceiver(ReceiverConfig{ + BindAddr: "127.0.0.1:0", Identity: serverIdentity, LogInterval: -1, + AdmitPeer: func(solana.PublicKey) bool { return true }, + }, NewObserver()) + require.NoError(t, err) + runVotorReceiver(t, r) + cert, err := newVotorQUICCertificate(ed25519.NewKeyFromSeed(bytesOf(182, ed25519.SeedSize))) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + peer := VotorPeer{Identity: testVotorPubkey(serverIdentity), Addr: r.Addr().(*net.UDPAddr)} + b := &VotorBroadcaster{ + ctx: ctx, cancel: cancel, done: make(chan struct{}), + tlsConfig: &tls.Config{Certificates: []tls.Certificate{cert}, NextProtos: []string{VotorQUICALPN}, MinVersion: tls.VersionTLS13, InsecureSkipVerify: true}, + quicConfig: newVotorQUICConfig(), + jobs: make(chan votorPeerJob, 2), desired: map[solana.PublicKey]VotorPeer{peer.Identity: peer}, + conns: make(map[solana.PublicKey]votorConnection), dialing: make(map[solana.PublicKey]*votorDial), connectQueued: make(map[solana.PublicKey]struct{}), + } + t.Cleanup(func() { require.NoError(t, b.Close()) }) + _, err = b.connection(peer) + require.NoError(t, err) + b.connMu.Lock() + sender := b.conns[peer.Identity].sender + b.connMu.Unlock() + return b, sender, r +} + +func TestVotorPeerReconnectOnRemoteCloseWithoutReconcile(t *testing.T) { + for _, duringDequeue := range []bool{false, true} { + name := "idle" + if duringDequeue { + name = "dequeue" + } + t.Run(name, func(t *testing.T) { + b, s, receiver := passiveVotorSender(t) + if duringDequeue { + func() { + s.mu.Lock() + defer s.mu.Unlock() + s.queue <- votorDatagram{payload: []byte{1}, queuedAt: time.Now()} + // Force the job branch to win select, then close remotely + // while the sender is waiting to check the connection state. + require.Eventually(t, func() bool { return len(s.queue) == 0 }, time.Second, time.Millisecond) + require.NoError(t, receiver.Close()) + select { + case <-s.conn.Context().Done(): + case <-time.After(time.Second): + t.Fatal("remote close not observed") + } + }() + } else { + require.NoError(t, receiver.Close()) + } + select { + case <-s.done: + case <-time.After(time.Second): + t.Fatal("sender did not exit") + } + select { + case job := <-b.jobs: + require.Equal(t, s.peer.Identity, job.peer.Identity) + default: + t.Fatal("sender exited without requesting reconnect") + } + require.Empty(t, b.jobs, "only one reconnect request per peer") + }) + } +} + +func TestVotorPeerDeadlineIncludesQueueAge(t *testing.T) { + for _, tc := range []struct { + name string + queued, sending time.Duration + active, expired bool + }{ + {"progress_does_not_reset_age", 950 * time.Millisecond, 100 * time.Millisecond, true, true}, + {"below_deadline", 800 * time.Millisecond, 100 * time.Millisecond, true, false}, + {"blocked_call", 0, time.Second, true, true}, + {"completed_send", 2 * time.Second, 0, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + b, s, _ := passiveVotorSender(t) + now := time.Now() + s.mu.Lock() + if tc.active { + s.sendingSince = now.Add(-tc.sending) + } + s.lastQueueDelay = tc.queued + s.mu.Unlock() + // Inject a deterministic clock/state boundary. There is no actual + // datagram in progress and no timer goroutine in this fixture. + b.expirePeerSends(now) + require.Equal(t, tc.expired, s.conn.Context().Err() != nil) + if tc.expired { + require.EqualValues(t, 1, b.peerSendTimeouts.Load()) + b.expirePeerSends(now.Add(time.Second)) + require.EqualValues(t, 1, b.peerSendTimeouts.Load()) + } else { + require.Zero(t, b.peerSendTimeouts.Load()) + } + }) + } +} + +func TestVotorPeerRejectsAgedQueueBeforeQUICEnqueue(t *testing.T) { + b, s, _ := passiveVotorSender(t) + s.enqueue(votorDatagram{payload: []byte{1}, queuedAt: time.Now().Add(-2 * time.Second)}) + select { + case <-s.done: + case <-time.After(time.Second): + t.Fatal("aged queue did not retire its connection") + } + require.Error(t, s.conn.Context().Err()) + require.Zero(t, b.sends.Load(), "stale backlog must not enter the QUIC queue") + require.EqualValues(t, 1, b.peerSendTimeouts.Load()) + require.EqualValues(t, 1, b.peerQueueDiscarded.Load()) + require.Len(t, b.jobs, 1) +} diff --git a/pkg/alpenglow/peer_sender_test.go b/pkg/alpenglow/peer_sender_test.go new file mode 100644 index 000000000..dda145c4a --- /dev/null +++ b/pkg/alpenglow/peer_sender_test.go @@ -0,0 +1,245 @@ +package alpenglow + +import ( + "crypto/ed25519" + "net" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Real QUIC traffic: one path is blackholed after authentication while the +// other stays healthy. This reproduces the original shared-worker failure. +func TestVotorBroadcasterIsolatesBlockedPeer(t *testing.T) { + for _, action := range []string{"reconnect", "close", "depart", "move"} { + t.Run(action, func(t *testing.T) { testVotorBlockedPeer(t, action) }) + } +} + +func testVotorBlockedPeer(t *testing.T, action string) { + const markerSlot = 999999 + marker := make(chan time.Time, 1) + badMarker := make(chan time.Time, 1) + makeReceiver := func(seed byte, record bool) (*Receiver, solana.PublicKey) { + identity := ed25519.NewKeyFromSeed(bytesOf(seed, ed25519.SeedSize)) + r, err := NewReceiver(ReceiverConfig{ + BindAddr: "127.0.0.1:0", Identity: identity, LogInterval: -1, + MaxDatagramsPerSecond: 100000, + AdmitPeer: func(solana.PublicKey) bool { return true }, + AdmitMessage: func(_ solana.PublicKey, m Message) (Message, bool) { + if m.Slot() == markerSlot { + target := badMarker + if record { + target = marker + } + select { + case target <- time.Now(): + default: + } + } + return Message{}, false + }, + }, NewObserver()) + require.NoError(t, err) + runVotorReceiver(t, r) + return r, testVotorPubkey(identity) + } + badReceiver, badID := makeReceiver(191, false) + goodReceiver, goodID := makeReceiver(192, true) + badAddr := badReceiver.Addr().(*net.UDPAddr) + proxy, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + var blackhole atomic.Bool + proxyDone := make(chan struct{}) + go func() { + defer close(proxyDone) + buf := make([]byte, 65536) + var client *net.UDPAddr + for { + n, from, err := proxy.ReadFromUDP(buf) + if err != nil { + return + } + if blackhole.Load() { + continue + } + if from.String() == badAddr.String() { + if client != nil { + _, _ = proxy.WriteToUDP(buf[:n], client) + } + } else { + client = from + _, _ = proxy.WriteToUDP(buf[:n], badAddr) + } + } + }() + t.Cleanup(func() { _ = proxy.Close(); <-proxyDone }) + badPeer := VotorPeer{Identity: badID, Addr: proxy.LocalAddr().(*net.UDPAddr)} + goodPeer := VotorPeer{Identity: goodID, Addr: goodReceiver.Addr().(*net.UDPAddr)} + peers := newMutableVotorPeers([]VotorPeer{badPeer, goodPeer}) + b, err := NewVotorBroadcaster(VotorBroadcasterConfig{ + Identity: ed25519.NewKeyFromSeed(bytesOf(193, ed25519.SeedSize)), + Peers: peers.Snapshot, + Workers: defaultVotorConnectWorkers, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, b.Close()) }) + require.Eventually(t, func() bool { return b.Stats().Connections == 2 }, 3*time.Second, 5*time.Millisecond) + badConn, ok := b.establishedConnection(badPeer) + require.True(t, ok) + // Establish an actual healthy delivery before introducing the fault. + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x41), 3))) + select { + case <-marker: + case <-time.After(time.Second): + t.Fatal("healthy baseline failed") + } + select { + case <-badMarker: + case <-time.After(time.Second): + t.Fatal("proxied baseline failed") + } + blackholedAt := time.Now() + blackhole.Store(true) + for slot := uint64(1); slot <= 256; slot++ { + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(slot), testSignatureSeq(0x41), 3))) + } + blockedWorkers := func() int { + buf := make([]byte, 2<<20) + stack := string(buf[:runtime.Stack(buf, true)]) + count := 0 + for _, goroutine := range strings.Split(stack, "\n\n") { + if strings.Contains(goroutine, "(*datagramQueue).Add") && strings.Contains(goroutine, "(*votorPeerSender).run") { + count++ + } + } + return count + } + require.Eventually(t, func() bool { return blockedWorkers() == 1 }, 3*time.Second, 5*time.Millisecond) + before := b.Stats() + require.Zero(t, before.MessagesDropped) + goodConn, ok := b.establishedConnection(goodPeer) + require.True(t, ok) + b.connMu.Lock() + badSender := b.conns[badID].sender + b.connMu.Unlock() + start := time.Now() + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x42), 3))) + select { + case received := <-marker: + t.Logf("Healthy marker received in %s while other peer's SendDatagram is blocked", received.Sub(start)) + case <-time.After(250 * time.Millisecond): + t.Fatal("stalled peer delayed healthy delivery") + } + require.NoError(t, badConn.Context().Err(), "marker must arrive before watchdog releases stalled peer") + if action != "reconnect" { + switch action { + case "close": + closed := make(chan struct{}) + go func() { _ = b.Close(); close(closed) }() + select { + case <-closed: + case <-time.After(500 * time.Millisecond): + t.Fatal("Close waited for stalled SendDatagram") + } + case "depart": + peers.Set([]VotorPeer{goodPeer}) + b.reconcilePeers() + case "move": + replacement, _ := makeReceiver(191, false) + badPeer.Addr = replacement.Addr().(*net.UDPAddr) + peers.Set([]VotorPeer{badPeer, goodPeer}) + b.reconcilePeers() + } + select { + case <-badSender.done: + case <-time.After(500 * time.Millisecond): + t.Fatal("old sender did not stop") + } + require.Error(t, badConn.Context().Err()) + require.Empty(t, badSender.queue) + require.Positive(t, b.Stats().PeerQueueDiscarded) + if action == "close" { + return + } + if action == "move" { + require.Eventually(t, func() bool { + conn, ok := b.establishedConnection(badPeer) + return ok && conn != badConn + }, 3*time.Second, 5*time.Millisecond) + } + currentGood, ok := b.establishedConnection(goodPeer) + require.True(t, ok) + require.Same(t, goodConn, currentGood) + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x44), 3))) + select { + case <-marker: + case <-time.After(time.Second): + t.Fatal("healthy delivery stopped after peer change") + } + if action == "move" { + select { + case <-badMarker: + case <-time.After(time.Second): + t.Fatal("replacement address did not receive new vote") + } + } + return + } + // Deliberately fill just the stalled peer's bounded queue. Healthy peers + // remain independent even when the failed peer's copies are rejected. + payload, err := EncodeMessage(NewVoteMessage(NewSkipVote(123), testSignatureSeq(0x42), 3)) + require.NoError(t, err) + for range 2 * defaultVotorPeerSendQueue { + badSender.enqueue(votorDatagram{payload: payload, queuedAt: time.Now()}) + } + require.Equal(t, defaultVotorPeerSendQueue, len(badSender.queue)) + require.Positive(t, b.Stats().PeerQueueDrops) + require.Equal(t, b.Stats().PeerQueueDrops, b.Stats().MessagesDropped) + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x43), 3))) + select { + case <-marker: + case <-time.After(250 * time.Millisecond): + t.Fatal("full peer queue delayed healthy delivery") + } + // Queue age is measured from fanout, so PTO progress no longer restarts + // the one-second budget. Allow two seconds of test scheduling slack beyond + // one watchdog tick, measured from the fault rather than this assertion. + remaining := time.Until(blackholedAt.Add(votorSendTimeout + votorSendWatchInterval + 2*time.Second)) + require.Positive(t, remaining) + require.Eventually(t, func() bool { return badConn.Context().Err() != nil }, remaining, 5*time.Millisecond) + t.Logf("Blackholed peer retired after %s", time.Since(blackholedAt)) + select { + case <-badSender.done: + case <-time.After(time.Second): + t.Fatal("stalled sender leaked after watchdog closed its connection") + } + require.EqualValues(t, 1, b.Stats().PeerSendTimeouts) + require.Positive(t, b.Stats().PeerQueueDiscarded) + require.Empty(t, badSender.queue) + // The old queue must never be resurrected on a replacement connection. + blackhole.Store(false) + require.Eventually(t, func() bool { + conn, ok := b.establishedConnection(badPeer) + return ok && conn != badConn + }, 5*time.Second, 10*time.Millisecond) + currentGood, ok := b.establishedConnection(goodPeer) + require.True(t, ok) + require.Same(t, goodConn, currentGood) + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x44), 3))) + select { + case <-marker: + case <-time.After(time.Second): + t.Fatal("healthy peer did not continue after reconnect") + } + select { + case <-badMarker: + case <-time.After(time.Second): + t.Fatal("reconnected peer did not receive new vote") + } +} diff --git a/pkg/alpenglow/testdata/README.md b/pkg/alpenglow/testdata/README.md index 92b75d4eb..10997bf71 100644 --- a/pkg/alpenglow/testdata/README.md +++ b/pkg/alpenglow/testdata/README.md @@ -12,3 +12,12 @@ The v4.3 vote wire message intentionally excludes validator rank and stake; the authenticated Votor transport identity supplies those values after decode. The shred version is the final little-endian `u16`. Certificate bitmap vectors use wincode's default bincode-compatible little-endian `u64` length. + +`agave_votor_certificate.der` is the 249-byte certificate constructed by +`anza-xyz/agave` commit `8fe3f1201abc5b0244540aed0c7bf8c6bcafb3f5`, +`tls-utils/src/tls_certificates.rs::new_dummy_x509_certificate`, with public +key bytes `00..1f` at offsets 100–131. It also matches Firedancer commit +`de039cd7fc9f4714782ec7e3d47db3728903abc6`, +`src/ballet/x509/fd_x509_mock.c::fd_x509_mock_pubkey_v2`. Its fixed dummy +X.509 signature is intentional; TLS CertificateVerify proves possession of +the identity key. This fixture contains no private key. diff --git a/pkg/alpenglow/testdata/agave_votor_certificate.der b/pkg/alpenglow/testdata/agave_votor_certificate.der new file mode 100644 index 000000000..e23bc0bfb Binary files /dev/null and b/pkg/alpenglow/testdata/agave_votor_certificate.der differ diff --git a/pkg/alpenglow/tls_identity.go b/pkg/alpenglow/tls_identity.go index 1e8b91d91..3c18e0cfd 100644 --- a/pkg/alpenglow/tls_identity.go +++ b/pkg/alpenglow/tls_identity.go @@ -5,9 +5,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "fmt" - "math/big" "time" "github.com/gagliardetto/solana-go" @@ -27,6 +25,36 @@ func newVotorQUICConfig() *quic.Config { } } +// votorCertificateTemplate is Agave's dummy X.509 certificate from +// tls-utils/src/tls_certificates.rs (8fe3f1201abc5b0244540aed0c7bf8c6bcafb3f5). +// Firedancer's fd_x509_mock_pubkey_v2 requires this exact encoding except for +// the 32-byte SubjectPublicKeyInfo at offset 100. The X.509 signature is +// deliberately invalid: TLS 1.3 CertificateVerify authenticates the identity. +// Keep the template unchanged and copy it before inserting a public key. +var votorCertificateTemplate = [...]byte{ + 0x30, 0x81, 0xf6, 0x30, 0x81, 0xa9, 0xa0, 0x03, 0x02, 0x01, 0x02, 0x02, + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x30, 0x05, 0x06, + 0x03, 0x2b, 0x65, 0x70, 0x30, 0x16, 0x31, 0x14, 0x30, 0x12, 0x06, 0x03, + 0x55, 0x04, 0x03, 0x0c, 0x0b, 0x53, 0x6f, 0x6c, 0x61, 0x6e, 0x61, 0x20, + 0x6e, 0x6f, 0x64, 0x65, 0x30, 0x20, 0x17, 0x0d, 0x37, 0x30, 0x30, 0x31, + 0x30, 0x31, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x5a, 0x18, 0x0f, 0x34, + 0x30, 0x39, 0x36, 0x30, 0x31, 0x30, 0x31, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x5a, 0x30, 0x00, 0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, + 0x70, 0x03, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa3, 0x29, 0x30, 0x27, 0x30, 0x17, 0x06, 0x03, 0x55, 0x1d, 0x11, 0x01, + 0x01, 0xff, 0x04, 0x0d, 0x30, 0x0b, 0x82, 0x09, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x01, 0x01, 0xff, 0x04, 0x02, 0x30, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, + 0x65, 0x70, 0x03, 0x41, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +} + // newVotorQUICCertificate creates the single-certificate Ed25519 identity // chain used by the Agave Votor transport. Peers recover the validator identity // directly from the leaf certificate's SubjectPublicKeyInfo; TLS 1.3's @@ -47,23 +75,8 @@ func newVotorQUICCertificate(identity ed25519.PrivateKey) (tls.Certificate, erro pub = priv.Public().(ed25519.PublicKey) } - template := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: "Mithril Alpenglow observer", - }, - NotBefore: time.Unix(0, 0), - NotAfter: time.Date(4096, 1, 1, 0, 0, 0, 0, time.UTC), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - DNSNames: []string{"localhost"}, - BasicConstraintsValid: true, - } - - certDER, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) - if err != nil { - return tls.Certificate{}, err - } + certDER := append([]byte(nil), votorCertificateTemplate[:]...) + copy(certDER[100:100+ed25519.PublicKeySize], pub) cert, err := x509.ParseCertificate(certDER) if err != nil { return tls.Certificate{}, err diff --git a/pkg/alpenglow/tls_identity_test.go b/pkg/alpenglow/tls_identity_test.go index 7efaa4125..cb591e0fa 100644 --- a/pkg/alpenglow/tls_identity_test.go +++ b/pkg/alpenglow/tls_identity_test.go @@ -1,15 +1,61 @@ package alpenglow import ( + "context" "crypto/ed25519" "crypto/tls" "crypto/x509" + "net" + "os" "testing" "time" "github.com/stretchr/testify/require" ) +func TestVotorCertificateMatchesAgaveAndFiredancerTemplate(t *testing.T) { + // Independent fixture from Agave's constructor, also accepted by + // Firedancer's fd_x509_mock_pubkey_v2 byte-pattern parser. + fixture, err := os.ReadFile("testdata/agave_votor_certificate.der") + require.NoError(t, err) + require.Len(t, fixture, 249) + for _, seed := range []byte{61, 62} { + identity := ed25519.NewKeyFromSeed(bytesOf(seed, ed25519.SeedSize)) + certificate, err := newVotorQUICCertificate(identity) + require.NoError(t, err) + expected := append([]byte(nil), fixture...) + copy(expected[100:132], identity.Public().(ed25519.PublicKey)) + require.Equal(t, expected, certificate.Certificate[0]) + require.Equal(t, identity.Public(), certificate.Leaf.PublicKey) + // Do not replace the dummy signature with a real one: Firedancer's + // parser matches it too. Authentication happens in CertificateVerify. + require.Error(t, certificate.Leaf.CheckSignature(certificate.Leaf.SignatureAlgorithm, + certificate.Leaf.RawTBSCertificate, certificate.Leaf.Signature)) + } +} + +func TestVotorCertificateStillRequiresIdentityKeyPossession(t *testing.T) { + identity := ed25519.NewKeyFromSeed(bytesOf(63, ed25519.SeedSize)) + certificate, err := newVotorQUICCertificate(identity) + require.NoError(t, err) + certificate.PrivateKey = ed25519.NewKeyFromSeed(bytesOf(64, ed25519.SeedSize)) + serverConn, clientConn := net.Pipe() + t.Cleanup(func() { _ = serverConn.Close(); _ = clientConn.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + server := tls.Server(serverConn, &tls.Config{ + Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS13, + }) + client := tls.Client(clientConn, &tls.Config{ + InsecureSkipVerify: true, MinVersion: tls.VersionTLS13, + }) + serverDone := make(chan error, 1) + go func() { serverDone <- server.HandshakeContext(ctx) }() + // Skipping the dummy X.509 signature does not bypass CertificateVerify. + require.ErrorContains(t, client.HandshakeContext(ctx), "invalid signature") + require.Error(t, <-serverDone) +} + func TestVotorPeerIdentityRequiresOneEd25519Certificate(t *testing.T) { identity := ed25519.NewKeyFromSeed(bytesOf(61, ed25519.SeedSize)) certificate, err := newVotorQUICCertificate(identity) diff --git a/pkg/alpenglow/vote_history.go b/pkg/alpenglow/vote_history.go index d227bbe12..448977fa3 100644 --- a/pkg/alpenglow/vote_history.go +++ b/pkg/alpenglow/vote_history.go @@ -15,6 +15,7 @@ import ( ) const voteHistoryVersion = 1 +const reservedVoteHistoryVersion = 2 var ErrVoteHistoryNotFound = errors.New("alpenglow vote history not found") @@ -23,6 +24,7 @@ var ErrVoteHistoryNotFound = errors.New("alpenglow vote history not found") // Agave when resuming (notarized blocks and ParentReady edges), in addition to // the anti-equivocation vote sets. type VoteHistory struct { + ReservationRequired bool `json:"reservation_required,omitempty"` Version uint32 `json:"version"` NodePubkey solana.PublicKey `json:"node_pubkey"` Root uint64 `json:"root"` @@ -420,50 +422,111 @@ func VoteHistoryFilename(dir string, node solana.PublicKey) string { return filepath.Join(dir, fmt.Sprintf("vote_history-%s.mithril.json", node)) } -// SaveVoteHistory signs the exact serialized history with the validator -// identity and atomically replaces the previous file before a vote can be -// admitted to consensus or sent to the network. +// SaveVoteHistory authenticates and durably replaces the exact history: write, +// file sync, rename, then directory sync. Synchronous-mode callers require +// success before pool admission (which can publish certificates) or network +// enqueue. The BLS signature may already have been computed privately in RAM; +// this is persist-before-publication, not persist-before-BLS-computation. func SaveVoteHistory(dir string, h *VoteHistory, identity ed25519.PrivateKey) error { + return saveVoteHistory(dir, h, identity, true) +} + +// SaveReservedVoteHistory writes and renames without per-vote sync. Success +// does not prove that this history survived a host/power failure. The caller +// must enforce an independently durable signing reservation and its restart +// quarantine; a valid-looking older history is not evidence of completeness. +func SaveReservedVoteHistory(dir string, h *VoteHistory, identity ed25519.PrivateKey) error { + if h == nil || !h.ReservationRequired { + return fmt.Errorf("reserved history requires durable reservation enrollment") + } + return saveVoteHistory(dir, h, identity, false) +} + +// VoteHistorySnapshot owns signed, immutable bytes. It retains no reference to +// the voter's mutable maps or signing key and can be saved by a worker. +type VoteHistorySnapshot struct { + node solana.PublicKey + encoded []byte +} + +// PrepareReservedVoteHistory validates and signs the complete current history +// without filesystem access. Only the owner of h may call this while mutating h. +func PrepareReservedVoteHistory(h *VoteHistory, identity ed25519.PrivateKey) (*VoteHistorySnapshot, error) { + if h == nil || !h.ReservationRequired { + return nil, fmt.Errorf("reserved history requires durable reservation enrollment") + } + encoded, err := encodeVoteHistory(h, identity) + if err != nil { + return nil, err + } + return &VoteHistorySnapshot{node: h.NodePubkey, encoded: encoded}, nil +} + +// SaveReservedVoteHistorySnapshot replaces history without an explicit sync. +// Success means replacement completed, not durable vote acknowledgement. Callers +// must serialize writes and enforce the independent durable reservation. On an +// unclean restart even an intact snapshot cannot bypass the startup bound. +func SaveReservedVoteHistorySnapshot(dir string, snapshot *VoteHistorySnapshot) error { + if snapshot == nil || len(snapshot.encoded) == 0 { + return fmt.Errorf("save reserved history: empty snapshot") + } + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return err + } + return replaceVoteHistoryFile(dir, VoteHistoryFilename(dir, snapshot.node), snapshot.encoded, false) +} + +func saveVoteHistory(dir string, h *VoteHistory, identity ed25519.PrivateKey, durable bool) error { + encoded, err := encodeVoteHistory(h, identity) + if err != nil { + return err + } + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return err + } + return replaceVoteHistoryFile(dir, VoteHistoryFilename(dir, h.NodePubkey), encoded, durable) +} + +func encodeVoteHistory(h *VoteHistory, identity ed25519.PrivateKey) ([]byte, error) { if h == nil { - return fmt.Errorf("save vote history: nil history") + return nil, fmt.Errorf("save vote history: nil history") } if len(identity) != ed25519.PrivateKeySize { - return fmt.Errorf("save vote history: invalid identity key size %d", len(identity)) + return nil, fmt.Errorf("save vote history: invalid identity key size %d", len(identity)) } node := solana.PublicKey(identity.Public().(ed25519.PublicKey)) if node != h.NodePubkey { - return fmt.Errorf("save vote history: identity %s does not match history %s", node, h.NodePubkey) + return nil, fmt.Errorf("save vote history: identity %s does not match history %s", node, h.NodePubkey) } h.Version = voteHistoryVersion + if h.ReservationRequired { + h.Version = reservedVoteHistoryVersion + } if err := h.preparePersistedViews(); err != nil { - return fmt.Errorf("save vote history: %w", err) + return nil, fmt.Errorf("save vote history: %w", err) } defer func() { h.PersistedNotarized = nil h.PersistedParentReady = nil }() if err := h.validatePersistedState(); err != nil { - return fmt.Errorf("save vote history: %w", err) + return nil, fmt.Errorf("save vote history: %w", err) } data, err := json.Marshal(h) if err != nil { - return fmt.Errorf("serialize vote history: %w", err) + return nil, fmt.Errorf("serialize vote history: %w", err) } envelope := savedVoteHistory{ - Version: voteHistoryVersion, + Version: h.Version, Node: node, Data: data, Signature: ed25519.Sign(identity, data), } encoded, err := json.Marshal(envelope) if err != nil { - return fmt.Errorf("serialize saved vote history: %w", err) + return nil, fmt.Errorf("serialize saved vote history: %w", err) } - if err := ensureDurableVoteHistoryDirectory(dir); err != nil { - return err - } - filename := VoteHistoryFilename(dir, node) - return persistVoteHistoryFile(dir, filename, encoded) + return encoded, nil } // ensureDurableVoteHistoryDirectory creates each missing path component and @@ -515,6 +578,10 @@ func ensureDurableVoteHistoryDirectory(dir string) error { // alone does not guarantee that either the bytes or the new directory entry // survives a crash. func persistVoteHistoryFile(dir, filename string, encoded []byte) error { + return replaceVoteHistoryFile(dir, filename, encoded, true) +} + +func replaceVoteHistoryFile(dir, filename string, encoded []byte, durable bool) error { temporary, err := os.CreateTemp(dir, "."+filepath.Base(filename)+".tmp-") if err != nil { return fmt.Errorf("create temporary vote history: %w", err) @@ -538,8 +605,10 @@ func persistVoteHistoryFile(dir, filename string, encoded []byte) error { if n != len(encoded) { return fmt.Errorf("write temporary vote history: %w", io.ErrShortWrite) } - if err := temporary.Sync(); err != nil { - return fmt.Errorf("sync temporary vote history: %w", err) + if durable { + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary vote history: %w", err) + } } closeErr := temporary.Close() closed = true @@ -551,8 +620,8 @@ func persistVoteHistoryFile(dir, filename string, encoded []byte) error { } renamed = true - if err := syncVoteHistoryDirectory(dir); err != nil { - return err + if durable { + return syncVoteHistoryDirectory(dir) } return nil } @@ -586,7 +655,7 @@ func LoadVoteHistory(dir string, node solana.PublicKey) (*VoteHistory, error) { if err := json.Unmarshal(encoded, &envelope); err != nil { return nil, fmt.Errorf("decode saved vote history: %w", err) } - if envelope.Version != voteHistoryVersion || envelope.Node != node { + if (envelope.Version != voteHistoryVersion && envelope.Version != reservedVoteHistoryVersion) || envelope.Node != node { return nil, fmt.Errorf("saved vote history identity/version mismatch") } if !ed25519.Verify(ed25519.PublicKey(node[:]), envelope.Data, envelope.Signature) { @@ -596,7 +665,7 @@ func LoadVoteHistory(dir string, node solana.PublicKey) (*VoteHistory, error) { if err := json.Unmarshal(envelope.Data, &h); err != nil { return nil, fmt.Errorf("decode vote history: %w", err) } - if h.Version != voteHistoryVersion || h.NodePubkey != node { + if h.Version != envelope.Version || h.NodePubkey != node || h.ReservationRequired != (h.Version == reservedVoteHistoryVersion) { return nil, fmt.Errorf("vote history identity/version mismatch") } if err := h.validatePersistedState(); err != nil { diff --git a/pkg/alpenglow/vote_history_snapshot_test.go b/pkg/alpenglow/vote_history_snapshot_test.go new file mode 100644 index 000000000..540e6fa14 --- /dev/null +++ b/pkg/alpenglow/vote_history_snapshot_test.go @@ -0,0 +1,49 @@ +package alpenglow + +import ( + "crypto/ed25519" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestReservedHistorySnapshotIsImmutable(t *testing.T) { + identity := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + node := solana.PublicKey(identity.Public().(ed25519.PublicKey)) + h := NewVoteHistory(node, 10) + h.ReservationRequired = true + block := BlockID{Slot: 11, Hash: solana.Hash{11}} + require.NoError(t, h.AddVote(NewNotarizationVote(11, block.Hash))) + h.NotarizedBlocks[block] = true + h.AddParentReady(12, block) + snapshot, err := PrepareReservedVoteHistory(h, identity) + require.NoError(t, err) + // Mutate/prune every transport-backed collection after taking the snapshot. + h.SetRoot(20) + require.NoError(t, h.AddVote(NewSkipVote(21))) + for i := range identity { + identity[i] = 0 + } + dir := t.TempDir() + require.NoError(t, SaveReservedVoteHistorySnapshot(dir, snapshot)) + loaded, err := LoadVoteHistory(dir, node) + require.NoError(t, err) + require.Equal(t, uint64(10), loaded.Root) + require.True(t, loaded.VotedAt(11)) + require.True(t, loaded.IsBlockNotarized(block)) + require.True(t, loaded.IsParentReady(12, block)) + require.False(t, loaded.HasSkipped(21)) +} + +func TestReservedHistorySnapshotRequiresEnrollmentAndValidHistory(t *testing.T) { + identity := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + h := NewVoteHistory(solana.PublicKey(identity.Public().(ed25519.PublicKey)), 10) + _, err := PrepareReservedVoteHistory(h, identity) + require.Error(t, err) + h.ReservationRequired = true + h.Voted[11] = true // Inconsistent with canonical VotesCast. + _, err = PrepareReservedVoteHistory(h, identity) + require.Error(t, err) + require.Error(t, SaveReservedVoteHistorySnapshot(t.TempDir(), &VoteHistorySnapshot{})) +} diff --git a/pkg/alpenglow/vote_reservation.go b/pkg/alpenglow/vote_reservation.go new file mode 100644 index 000000000..b577da189 --- /dev/null +++ b/pkg/alpenglow/vote_reservation.go @@ -0,0 +1,104 @@ +package alpenglow + +import ( + "crypto/ed25519" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/gagliardetto/solana-go" + "golang.org/x/sys/unix" +) + +// VoteReservation is the durable upper bound on slots this identity may sign, +// not a record of slots actually signed. It must survive independently of +// AccountsDB checkpoints and must never be restored from an older snapshot. +// A signature authenticates this file; it does not prove freshness against +// rollback, nor does Generation provide an external monotonic counter. +// CleanHistoryDigest permits exact-history vote recovery only after validation +// and durable consumption by a dirty successor before signing. It does not +// certify complete leader-block history. See docs/reserved-vote-history.md. +type VoteReservation struct { + Version uint32 `json:"version"` + Node solana.PublicKey `json:"node"` + VoteAccount solana.PublicKey `json:"vote_account"` + AuthorizedVoter solana.PublicKey `json:"authorized_voter"` + Genesis solana.Hash `json:"genesis"` + ShredVersion uint16 `json:"shred_version"` + Generation uint64 `json:"generation"` + Through uint64 `json:"through"` + CleanHistoryDigest []byte `json:"clean_history_digest,omitempty"` +} + +func VoteReservationFilename(dir string, node solana.PublicKey) string { + return filepath.Join(dir, fmt.Sprintf("vote_reservation-%s.mithril.json", node)) +} + +// LockVoteHistory excludes concurrent owners of this directory. Operators must +// still fence copies of the same identity on other hosts or in other paths. +func LockVoteHistory(dir string, node solana.PublicKey) (*os.File, error) { + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, ".vote_history-"+node.String()+".lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("vote history already owned: %w", err) + } + return f, nil +} + +func LoadVoteReservation(dir string, node solana.PublicKey) (VoteReservation, error) { + var r VoteReservation + encoded, err := os.ReadFile(VoteReservationFilename(dir, node)) + if err != nil { + return r, err + } + var envelope savedVoteHistory + if err := json.Unmarshal(encoded, &envelope); err != nil { + return r, err + } + if envelope.Version != 1 || envelope.Node != node || !ed25519.Verify(ed25519.PublicKey(node[:]), envelope.Data, envelope.Signature) { + return r, errors.New("invalid vote reservation signature/version/identity") + } + if err := json.Unmarshal(envelope.Data, &r); err != nil { + return r, err + } + if r.Version != 1 || r.Node != node || r.Generation == 0 || (len(r.CleanHistoryDigest) != 0 && len(r.CleanHistoryDigest) != sha256.Size) { + return r, errors.New("invalid vote reservation record") + } + return r, nil +} + +func SaveVoteReservation(dir string, r VoteReservation, identity ed25519.PrivateKey) error { + if len(identity) != ed25519.PrivateKeySize || solana.PublicKey(identity.Public().(ed25519.PublicKey)) != r.Node || r.Version != 1 || r.Generation == 0 { + return errors.New("invalid vote reservation signer/record") + } + data, err := json.Marshal(r) + if err != nil { + return err + } + encoded, err := json.Marshal(savedVoteHistory{Version: 1, Node: r.Node, Data: data, Signature: ed25519.Sign(identity, data)}) + if err != nil { + return err + } + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return err + } + return persistVoteHistoryFile(dir, VoteReservationFilename(dir, r.Node), encoded) +} + +func VoteHistoryDigest(dir string, node solana.PublicKey) ([]byte, error) { + data, err := os.ReadFile(VoteHistoryFilename(dir, node)) + if err != nil { + return nil, err + } + digest := sha256.Sum256(data) + return digest[:], nil +} diff --git a/pkg/alpenglow/vote_reservation_test.go b/pkg/alpenglow/vote_reservation_test.go new file mode 100644 index 000000000..9e1840ade --- /dev/null +++ b/pkg/alpenglow/vote_reservation_test.go @@ -0,0 +1,71 @@ +package alpenglow + +import ( + "crypto/ed25519" + "encoding/json" + "os" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestReservedHistoryFormatAndIntegrity(t *testing.T) { + key := ed25519.NewKeyFromSeed(make([]byte, 32)) + node := solana.PublicKey(key.Public().(ed25519.PublicKey)) + dir := t.TempDir() + h := NewVoteHistory(node, 39) + require.Error(t, SaveReservedVoteHistory(dir, h, key)) + h.ReservationRequired = true + require.NoError(t, h.AddVote(NewSkipVote(44))) + require.NoError(t, SaveReservedVoteHistory(dir, h, key)) + raw, err := os.ReadFile(VoteHistoryFilename(dir, node)) + require.NoError(t, err) + var envelope savedVoteHistory + require.NoError(t, json.Unmarshal(raw, &envelope)) + require.Equal(t, uint32(2), envelope.Version, "legacy reader must reject hybrid history") + loaded, err := LoadVoteHistory(dir, node) + require.NoError(t, err) + require.True(t, loaded.HasSkipped(44)) + require.True(t, loaded.ReservationRequired) + envelope.Data[10] ^= 1 + raw, err = json.Marshal(envelope) + require.NoError(t, err) + require.NoError(t, os.WriteFile(VoteHistoryFilename(dir, node), raw, 0600)) + _, err = LoadVoteHistory(dir, node) + require.Error(t, err) +} + +func BenchmarkVoteHistoryPersistence(b *testing.B) { + key := ed25519.NewKeyFromSeed(make([]byte, 32)) + node := solana.PublicKey(key.Public().(ed25519.PublicKey)) + for _, reserved := range []bool{false, true} { + name := "synchronous" + if reserved { + name = "reserved-write-rename" + } + b.Run(name, func(b *testing.B) { + dir := b.TempDir() + h := NewVoteHistory(node, 39) + h.ReservationRequired = reserved + for slot := uint64(40); slot < 72; slot++ { + if err := h.AddVote(NewNotarizationVote(slot, solana.Hash{byte(slot)})); err != nil { + b.Fatal(err) + } + } + save := SaveVoteHistory + if reserved { + save = SaveReservedVoteHistory + } + if err := SaveVoteHistory(dir, h, key); err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := save(dir, h, key); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/blockprod/leader.go b/pkg/blockprod/leader.go index e304ffb8d..65c062173 100644 --- a/pkg/blockprod/leader.go +++ b/pkg/blockprod/leader.go @@ -97,6 +97,7 @@ type LeaderLoop struct { alpenglowClock bool parentContext func(uint64) ParentContext productionParent func(uint64) alpenglow.BlockProductionParent + canSignSlot func(uint64) bool onBlock func(*b.Block) commitLeaderSlot func(replay.CommitLeaderInput) (*sealevel.SlotCtx, error) @@ -145,6 +146,7 @@ type LeaderLoopConfig struct { AlpenglowClock bool ParentContext func(uint64) ParentContext ProductionParent func(slot uint64) alpenglow.BlockProductionParent + CanSignSlot func(slot uint64) bool OnBlock func(*b.Block) CurrentSlot func() uint64 LeaderForSlot func(uint64) (solana.PublicKey, bool) @@ -181,6 +183,7 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { alpenglowClock: cfg.AlpenglowClock, parentContext: cfg.ParentContext, productionParent: cfg.ProductionParent, + canSignSlot: cfg.CanSignSlot, onBlock: cfg.OnBlock, currentSlot: cfg.CurrentSlot, leaderForSlot: cfg.LeaderForSlot, @@ -1101,6 +1104,9 @@ func (l *LeaderLoop) revalidateProductionParentForStartLocked(slot uint64, selec } func (l *LeaderLoop) startSlotLocked(slot uint64) error { + if l.canSignSlot != nil && !l.canSignSlot(slot) { + return fmt.Errorf("%w: waiting for durable signing reservation", errParentNotReady) + } selectedParent, parentReadyRequired, err := l.resolveProductionParent(slot) if err != nil { return err diff --git a/pkg/blockprod/leader_signing_reservation_test.go b/pkg/blockprod/leader_signing_reservation_test.go new file mode 100644 index 000000000..4b7e09c10 --- /dev/null +++ b/pkg/blockprod/leader_signing_reservation_test.go @@ -0,0 +1,17 @@ +package blockprod + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestSigningReservationGatesEveryLeaderSlotBeforeBuild(t *testing.T) { + for slot := uint64(40); slot < 44; slot++ { + var checked uint64 + l := &LeaderLoop{canSignSlot: func(s uint64) bool { checked = s; return false }} + require.ErrorIs(t, l.startSlotLocked(slot), errParentNotReady) + require.Equal(t, slot, checked) + // All other builder dependencies are deliberately nil: rejection must happen + // before accessing a working bank, executing or signing any shreds. + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e7416c90..c31bf7448 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -25,6 +25,7 @@ func ApplyDefaults(v *viper.Viper) { v.SetDefault("validator.tpu_quic_bind_addr", "") v.SetDefault("validator.advertised_ip", "") v.SetDefault("validator.tpu_sigverify_workers", 0) + v.SetDefault("validator.wait_to_vote_slot", uint64(0)) } // LedgerConfig holds ledger-related configuration (matches Firedancer [ledger] section) @@ -235,6 +236,7 @@ type ValidatorConfig struct { TPUQUICBindAddr string `toml:"tpu_quic_bind_addr" mapstructure:"tpu_quic_bind_addr"` AdvertisedIP string `toml:"advertised_ip" mapstructure:"advertised_ip"` TPUSigverifyWorkers int `toml:"tpu_sigverify_workers" mapstructure:"tpu_sigverify_workers"` + WaitToVoteSlot uint64 `toml:"wait_to_vote_slot" mapstructure:"wait_to_vote_slot"` // Minimum slot for new votes; automatic startup cutoff still applies } // Config holds all configuration options for Mithril (Firedancer-style hierarchy) diff --git a/pkg/consensus/engine.go b/pkg/consensus/engine.go index f57cb2ab5..5cf2cb203 100644 --- a/pkg/consensus/engine.go +++ b/pkg/consensus/engine.go @@ -389,7 +389,7 @@ func (e *AlpenglowObserverEngine) EnableVoting(cfg VotingConfig) error { // events, matching Agave's initial_parent_ready selection. if slot, parent, ok := voter.history.HighestParentReadyMatching(func(parent alpenglow.BlockID) bool { return !e.ensureChain().IsObjectivelyInvalidBlock(parent) - }); ok && slot > root.Slot { + }); ok && slot > root.Slot && (voter.reservation == nil || voter.reservation.recoverThrough == 0) { if !e.ensurePool().RestoreParentReady(slot, parent) { mlog.Log.FileOnlyf("ALPENGLOW voting: ignored persisted ParentReady slot=%d parent=%s because newer root/live tracker state is authoritative", slot, parent) } @@ -975,11 +975,18 @@ func (e *AlpenglowObserverEngine) injectLocalVote(message alpenglow.VoteMessage, } } -// alpenglowVoteActionFloor is the highest slot on which this validator must -// not initiate a new vote. The pool root is its strict admission boundary; -// direct finality is included because the pool deliberately retains a short -// reward-accounting tail behind finality where network votes remain useful. +// alpenglowVoteActionFloor is the retained pool's strict admission boundary. +// Network finality is not a voting root: a replayed block may still contribute +// a notarization to a later fast certificate and its slot+8 reward certificate. +// The voter also checks its own persisted history root before signing. func (e *AlpenglowObserverEngine) alpenglowVoteActionFloor() uint64 { + return e.ensurePool().Snapshot().RootSlot +} + +// alpenglowVerifiedFinalityFloor releases crash-recovery reservations. Keep this +// independent of live vote admission: a retained reward window must not weaken +// the requirement to pass every slot that may have been signed before a crash. +func (e *AlpenglowObserverEngine) alpenglowVerifiedFinalityFloor() uint64 { floor := e.ensurePool().Snapshot().RootSlot if finalized := e.ensureChain().Snapshot().LatestDirectFinalizedBlock.Slot; finalized > floor { floor = finalized @@ -1542,6 +1549,25 @@ func (e *AlpenglowObserverEngine) PruneAlpenglowBefore(slot uint64) { if slot == 0 { return } + // Replay enqueues its completed-block event before publishing a durable + // promotion. Retire the pool, execution proof and history on that same + // ordered voter stream, so a fast checkpoint cannot overtake the vote. + e.voterMu.RLock() + voter := e.voter + e.voterMu.RUnlock() + if voter != nil { + if err := voter.enqueue(voterEvent{kind: voterEventDurableRoot, slot: slot}); err != nil { + e.latchSafetyError(err) + } + return + } + e.applyAlpenglowDurableRoot(slot) +} + +// applyAlpenglowDurableRoot is called by the voter after earlier replay events, +// or synchronously by an observer without a voting loop. Startup root restore +// remains a separate, immediate barrier in SetAlpenglowRoot. +func (e *AlpenglowObserverEngine) applyAlpenglowDurableRoot(slot uint64) alpenglow.BlockID { e.poolOutputMu.Lock() defer e.poolOutputMu.Unlock() @@ -1557,13 +1583,11 @@ func (e *AlpenglowObserverEngine) PruneAlpenglowBefore(slot uint64) { if e.certPool != nil { e.certPool.ObserveFloor(slot) } - if err := e.enqueueVoter(voterEvent{kind: voterEventRoot, root: root}); err != nil { - e.latchSafetyError(err) - } // Replay calls this only after the fold through slot is durably committed. // Keep the transport peer window tied to that local root, not to speculative // certificate finality or the pool's reward-retention floor. e.advanceVotorPeerRoot(slot) + return root } func (e *AlpenglowObserverEngine) pruneInvalidBlockIDsBefore(slot uint64) { diff --git a/pkg/consensus/vote_history_writer.go b/pkg/consensus/vote_history_writer.go new file mode 100644 index 000000000..9d175d6d4 --- /dev/null +++ b/pkg/consensus/vote_history_writer.go @@ -0,0 +1,125 @@ +package consensus + +import ( + "errors" + "fmt" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" +) + +// One serial writer, one in-flight snapshot and at most one newer pending +// snapshot. A newer complete history supersedes an unwritten snapshot; every +// retained, unrooted voting decision is still present in that newer history. +// The independently durable reservation, not this queue, authorizes signing. +// In-flight/pending snapshots may be lost on process death; even a completed +// unsynced replacement may be lost on host/power failure. Neither submitted nor +// written is a durable vote acknowledgement. Recovery must use the startup +// reservation unless the separate clean-history seal validates. +type voteHistoryWriter struct { + mu sync.Mutex + pending *alpenglow.VoteHistorySnapshot + closing bool + failure error + submitted uint64 + written uint64 + coalesced uint64 + wake chan struct{} + done chan struct{} + persist func(*alpenglow.VoteHistorySnapshot) error + onError func(error) +} + +func newVoteHistoryWriter(persist func(*alpenglow.VoteHistorySnapshot) error, onError func(error)) *voteHistoryWriter { + w := &voteHistoryWriter{wake: make(chan struct{}, 1), done: make(chan struct{}), persist: persist, onError: onError} + go w.run() + return w +} + +// submit does no I/O and never waits for the writer. The mutex only protects +// pointer/counter changes; neither persistence nor error callbacks hold it. +// A nil return means queued only. It must never replace the reservation check. +func (w *voteHistoryWriter) submit(snapshot *alpenglow.VoteHistorySnapshot) error { + if snapshot == nil { + return errors.New("nil vote-history snapshot") + } + w.mu.Lock() + if w.failure != nil { + err := w.failure + w.mu.Unlock() + return err + } + if w.closing { + w.mu.Unlock() + return errors.New("vote-history writer is closed") + } + if w.pending != nil { + w.coalesced++ + } + w.pending = snapshot + w.submitted++ + w.mu.Unlock() + w.notify() + return nil +} + +func (w *voteHistoryWriter) notify() { + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *voteHistoryWriter) run() { + defer close(w.done) + for range w.wake { + for { + w.mu.Lock() + snapshot := w.pending + w.pending = nil + closing := w.closing + w.mu.Unlock() + if snapshot == nil { + if closing { + return + } + break + } + if err := w.persist(snapshot); err != nil { + err = fmt.Errorf("background vote-history write: %w", err) + w.mu.Lock() + w.failure = err + w.pending = nil + w.closing = true + w.mu.Unlock() + if w.onError != nil { + w.onError(err) + } + return + } + w.mu.Lock() + w.written++ + w.mu.Unlock() + } + } +} + +// close rejects new submissions and drains every retained snapshot. The voter +// must join this worker before writing and syncing its final clean history, +// otherwise an older in-flight rename could overwrite the sealed history. +func (w *voteHistoryWriter) close() error { + w.mu.Lock() + w.closing = true + w.mu.Unlock() + w.notify() + <-w.done + w.mu.Lock() + defer w.mu.Unlock() + return w.failure +} + +func (w *voteHistoryWriter) counters() (submitted, written, coalesced uint64) { + w.mu.Lock() + defer w.mu.Unlock() + return w.submitted, w.written, w.coalesced +} diff --git a/pkg/consensus/vote_history_writer_test.go b/pkg/consensus/vote_history_writer_test.go new file mode 100644 index 000000000..74c7bbfcd --- /dev/null +++ b/pkg/consensus/vote_history_writer_test.go @@ -0,0 +1,201 @@ +package consensus + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestAsyncHistoryBlockedWriteDoesNotDelayVotesAndCleanCloseDrains(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + require.NoError(t, v.historyWriter.close()) + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + t.Cleanup(unblock) + first := true // Owned only by the serial writer. + v.historyWriter = newVoteHistoryWriter(func(s *alpenglow.VoteHistorySnapshot) error { + if first { + first = false + close(entered) + <-release + } + return alpenglow.SaveReservedVoteHistorySnapshot(cfg.HistoryDir, s) + }, nil) + voted, err := v.cast(alpenglow.NewSkipVote(44), false) + require.NoError(t, err) + require.True(t, voted) + <-entered + castDone := make(chan error, 1) + go func() { + for _, slot := range []uint64{45, 46} { + ok, err := v.cast(alpenglow.NewSkipVote(slot), false) + if err != nil || !ok { + castDone <- errors.New("vote failed while history writer was blocked") + return + } + } + castDone <- nil + }() + select { + case err := <-castDone: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("disk writer blocked voting") + } + submitted, written, coalesced := v.historyWriter.counters() + require.Equal(t, uint64(3), submitted) + require.Zero(t, written) + require.Equal(t, uint64(1), coalesced) + onDisk, err := alpenglow.LoadVoteHistory(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.False(t, onDisk.HasSkipped(44), "I/O should still be blocked") + // Even when disk history lags, complete in-memory decisions forbid conflict. + voted, err = v.cast(alpenglow.NewNotarizationVote(44, solana.Hash{7}), false) + require.NoError(t, err) + require.False(t, voted) + closed := make(chan error, 1) + go func() { closed <- v.close() }() + require.Eventually(t, func() bool { + v.historyWriter.mu.Lock() + defer v.historyWriter.mu.Unlock() + return v.historyWriter.closing + }, time.Second, time.Millisecond) + select { + case err := <-closed: + t.Fatalf("close returned before draining its writer: %v", err) + default: + } + r, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) + unblock() + require.NoError(t, <-closed) + onDisk, err = alpenglow.LoadVoteHistory(cfg.HistoryDir, v.node) + require.NoError(t, err) + for _, slot := range []uint64{44, 45, 46} { + require.True(t, onDisk.HasSkipped(slot)) + } + r, err = alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + digest, err := alpenglow.VoteHistoryDigest(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Equal(t, digest, r.CleanHistoryDigest) + _, written, _ = v.historyWriter.counters() + require.Equal(t, uint64(2), written, "old in-flight snapshot must finish before newest complete snapshot") +} + +func TestAsyncHistoryFailureIsStickyAndReportedWithoutMoreSubmissions(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + h := alpenglow.NewVoteHistory(voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount).Validators[0].NodePubkey, 39) + h.ReservationRequired = true + snapshot, err := alpenglow.PrepareReservedVoteHistory(h, cfg.Identity) + require.NoError(t, err) + diskErr := errors.New("injected disk failure") + reported := make(chan error, 1) + w := newVoteHistoryWriter(func(*alpenglow.VoteHistorySnapshot) error { return diskErr }, func(err error) { reported <- err }) + require.NoError(t, w.submit(snapshot)) + select { + case err := <-reported: + require.ErrorIs(t, err, diskErr) + case <-time.After(time.Second): + t.Fatal("background failure was not reported") + } + require.ErrorIs(t, w.submit(snapshot), diskErr) + require.ErrorIs(t, w.close(), diskErr) + require.ErrorIs(t, w.close(), diskErr) +} + +func TestAsyncHistoryFailureStopsVoterAndPreventsCleanMarker(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + e, err := NewEngine(Config{AlpenglowIdentity: cfg.Identity, AlpenglowShredVersion: 0x1234}) + require.NoError(t, err) + t.Cleanup(func() { _ = e.Close() }) + set := voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount) + e.SetAlpenglowEpochLookup(cfg.EpochForSlot) + require.NoError(t, e.SetAlpenglowValidatorSet(set)) + root := alpenglow.BlockID{Slot: 39, Hash: solana.Hash{39}} + e.SetAlpenglowRoot(root) + v, err := newAlpenglowVoterUnstarted(e, cfg, root, []alpenglow.ValidatorSet{set}) + require.NoError(t, err) + t.Cleanup(func() { _ = v.close() }) + reserveThrough(t, v.reservation, 44) + require.NoError(t, v.historyWriter.close()) + diskErr := errors.New("injected background disk failure") + v.historyWriter = newVoteHistoryWriter(func(*alpenglow.VoteHistorySnapshot) error { return diskErr }, v.failHistoryWrite) + require.NoError(t, v.history.AddVote(alpenglow.NewSkipVote(44))) + require.NoError(t, v.saveHistory()) + select { + case <-v.done: + case <-time.After(time.Second): + t.Fatal("disk failure did not stop the voter") + } + require.ErrorIs(t, e.safetyError(), diskErr) + _, _, err = v.sign(alpenglow.NewSkipVote(45), false) + require.ErrorIs(t, err, diskErr) + require.Error(t, v.enqueue(voterEvent{kind: voterEventBlockTimeout, slot: 45})) + require.ErrorIs(t, v.close(), diskErr) + r, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) +} + +// Kill a subprocess while its first history write is blocked and a newer +// snapshot is pending. Successful earlier reservation syncs survive this +// process crash; this is deliberately not a host power-loss test. +func TestAsyncHistoryProcessCrashLosesPendingSnapshots(t *testing.T) { + const childEnv = "MITHRIL_ASYNC_HISTORY_TEST_DIR" + if dir := os.Getenv(childEnv); dir != "" { + cfg := reservedTestConfig(dir) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + require.NoError(t, v.historyWriter.close()) + entered := make(chan struct{}) + v.historyWriter = newVoteHistoryWriter(func(*alpenglow.VoteHistorySnapshot) error { + close(entered) + select {} + }, nil) + for _, slot := range []uint64{44, 45} { + voted, err := v.cast(alpenglow.NewSkipVote(slot), false) + require.NoError(t, err) + require.True(t, voted) + if slot == 44 { + <-entered + } + } + require.NoError(t, os.WriteFile(filepath.Join(dir, "ready"), []byte("ready"), 0600)) + select {} + } + dir := t.TempDir() + cmd := exec.Command(os.Args[0], "-test.run=^TestAsyncHistoryProcessCrashLosesPendingSnapshots$", "-test.count=1") + cmd.Env = append(os.Environ(), childEnv+"="+dir) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill() }) + require.Eventually(t, func() bool { _, err := os.Stat(filepath.Join(dir, "ready")); return err == nil }, 10*time.Second, time.Millisecond) + require.NoError(t, cmd.Process.Kill()) + require.Error(t, cmd.Wait()) + cfg := reservedTestConfig(dir) + cfg.InitializeVoteReservation = false + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.False(t, v.history.HasSkipped(44)) + require.False(t, v.history.HasSkipped(45)) + h := v.reservation.recoverThrough + require.GreaterOrEqual(t, h, uint64(45)) + _, _, err = v.sign(alpenglow.NewSkipVote(44), false) + require.ErrorIs(t, err, errVoterNotReady) + _, _, err = v.sign(alpenglow.NewSkipVote(h+1), false) + require.ErrorIs(t, err, errVoterNotReady, "verified finality must reach the lost history's bound") +} diff --git a/pkg/consensus/vote_reservation.go b/pkg/consensus/vote_reservation.go new file mode 100644 index 000000000..564cfc29d --- /dev/null +++ b/pkg/consensus/vote_reservation.go @@ -0,0 +1,296 @@ +package consensus + +import ( + "bytes" + "crypto/ed25519" + "errors" + "fmt" + "math" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/gagliardetto/solana-go" +) + +const signingReserveSlots = uint64(32) +const signingRenewRemaining = uint64(16) + +// signingReservation bounds what a crash may erase from detailed vote history. +// Intended guarantee: losing recent history must not authorize conflicting +// voting/leader actions after restart. It does NOT guarantee that every vote +// survives on disk, immediate restart voting, or recovery from safety-file rollback. +// +// On an enrolled restart, H is the startup reservation. Without a clean-history +// seal, all vote types (including restored votes) are forbidden at slots <= H; +// slots > H also wait until verified finality/checkpoint state reaches H. +// Leaders always obey that startup barrier, even after a clean vote-history seal. +// The current acknowledged Through separately caps every new signing permission. +// Renewal can raise Through, but never moves this run's fixed recovery barrier. +// +// The worker owns record after startup. Only a successful file+directory sync +// publishes Through to signers; a request, queued write or uncertain sync cannot. +// This assumes storage honors sync and a single fenced identity owner preserves +// the current reservation independently of AccountsDB. See docs/reserved-vote-history.md. +type signingReservation struct { + uncertain bool // Worker only, read after halt. Failed sync may have reached storage. + record alpenglow.VoteReservation + through atomic.Uint64 + desired atomic.Uint64 + stopped atomic.Bool + recoverThrough uint64 // Startup H, or zero after first enrollment / a validated clean-history seal. + leaderThrough uint64 // Exact leader production history is not saved: always skip the old range. + wake chan struct{} + changed chan struct{} + stop chan struct{} + done chan struct{} + stopOnce sync.Once + persist func(alpenglow.VoteReservation) error +} + +func openSigningReservation(cfg VotingConfig, node solana.PublicKey, shredVersion uint16, history *alpenglow.VoteHistory) (*signingReservation, error) { + if cfg.Genesis == (solana.Hash{}) { + return nil, errors.New("reserved voting requires the bound genesis hash") + } + expected := alpenglow.VoteReservation{Version: 1, Node: node, VoteAccount: cfg.VoteAccount, AuthorizedVoter: solana.PublicKey(cfg.AuthorizedVoter.Public().(ed25519.PublicKey)), Genesis: cfg.Genesis, ShredVersion: shredVersion} + record, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, node) + initializing := errors.Is(err, os.ErrNotExist) + if initializing { + if !cfg.InitializeVoteReservation || history.ReservationRequired { + return nil, errors.New("missing vote reservation; explicit first enrollment with complete synchronous history is required") + } + record = expected + record.Generation = 1 + record.Through = history.Root + for slot := range history.VotesCast { + record.Through = max(record.Through, slot) + } + // The baseline is made durable before the first reservation is created. + if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { + return nil, err + } + } else if err != nil { + return nil, fmt.Errorf("refuse unsafe reservation reset: %w", err) + } + if record.Node != expected.Node || record.VoteAccount != expected.VoteAccount || record.AuthorizedVoter != expected.AuthorizedVoter || record.Genesis != expected.Genesis || record.ShredVersion != expected.ShredVersion { + return nil, errors.New("vote reservation cluster or signing identity mismatch; explicit domain migration is required") + } + if record.Through == math.MaxUint64 || record.Generation == math.MaxUint64 { + return nil, errors.New("vote reservation exhausted") + } + for slot := range history.VotesCast { + if slot > record.Through { + return nil, fmt.Errorf("history slot %d exceeds durable reservation %d", slot, record.Through) + } + } + r := &signingReservation{record: record, recoverThrough: record.Through, leaderThrough: record.Through, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + r.persist = func(next alpenglow.VoteReservation) error { + return alpenglow.SaveVoteReservation(cfg.HistoryDir, next, cfg.Identity) + } + if initializing { + r.recoverThrough = 0 + } else if len(record.CleanHistoryDigest) != 0 { + digest, err := alpenglow.VoteHistoryDigest(cfg.HistoryDir, node) + if err != nil { + return nil, err + } + if bytes.Equal(digest, record.CleanHistoryDigest) { + r.recoverThrough = 0 + } + } + // A matching digest proves exact history only for the sealed session. Consume + // that exception with a durably acknowledged dirty successor before allowing + // new vote/leader signing or new detailed-history decisions. A crash after + // this write must use H, + // even if the detailed history file still looks valid or matches the old seal. + r.record.CleanHistoryDigest = nil + r.record.Generation++ + if err := r.persist(r.record); err != nil { + return nil, fmt.Errorf("consume vote reservation session: %w", err) + } + history.ReservationRequired = true + // Version 2 is deliberately rejected by older binaries that do not enforce H. + if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { + return nil, err + } + r.through.Store(r.record.Through) + mlog.Log.Infof("ALPENGLOW signing reservation: through=%d recovery_through=%d leader_recovery_through=%d", r.record.Through, r.recoverThrough, r.leaderThrough) + go r.run() + return r, nil +} + +// allow is nonblocking and protects vote signing/restoration and leader slots. +// For a nonzero startup barrier H, require slot > H AND finalized >= H. Merely +// observing a new block, waiting elapsed time, or replaying past H is not proof +// that prior decisions can be forgotten. Finality comes from verified consensus/ +// checkpoint state, never an RPC tip; --wait-to-vote-slot cannot override it. +// Passing this gate is necessary, not sufficient: normal protocol checks apply. +func (r *signingReservation) allow(slot, finalized uint64, leader bool) bool { + if r.stopped.Load() { + return false + } + floor := r.recoverThrough + if leader { + floor = r.leaderThrough + } + if floor != 0 && (slot <= floor || finalized < floor) { + return false + } + r.request(slot) + return slot <= r.through.Load() +} + +func (r *signingReservation) request(slot uint64) { + for { + old := r.desired.Load() + if slot <= old || r.desired.CompareAndSwap(old, slot) { + break + } + } + through := r.through.Load() + if slot > through || through-slot <= signingRenewRemaining { + select { + case r.wake <- struct{}{}: + default: + } + } +} + +func (r *signingReservation) run() { + defer close(r.done) + retry := time.NewTicker(250 * time.Millisecond) + defer retry.Stop() + var warned bool + for { + select { + case <-r.stop: + return + case <-r.wake: + case <-retry.C: + } + if r.stopped.Load() { + return + } + slot := r.desired.Load() + if slot == 0 || (slot <= r.record.Through && r.record.Through-slot > signingRenewRemaining) { + continue + } + if slot > math.MaxUint64-signingReserveSlots || r.record.Generation == math.MaxUint64 { + continue + } // Never wrap or grant permission. + next := r.record + next.Through = max(next.Through, slot+signingReserveSlots) + next.Generation++ + next.CleanHistoryDigest = nil + if err := r.persist(next); err != nil { + r.uncertain = true + if !warned { + mlog.Log.Errorf("ALPENGLOW signing reservation renewal failed; permission remains through %d: %v", r.through.Load(), err) + warned = true + } + continue // Retry the same or a greater bound; an uncertain sync never grants permission. + } + warned = false + r.uncertain = false + r.record = next + r.through.Store(next.Through) + select { + case r.changed <- struct{}{}: + default: + } + } +} + +func (r *signingReservation) halt() { + r.stopOnce.Do(func() { r.stopped.Store(true); close(r.stop) }) + <-r.done +} + +// seal may be called only after the voter loop and leader producer have stopped, +// and after the ordered history writer has been drained/joined. The caller must +// also establish verified finality >= recoverThrough and no latched safety fault. +// Sync exact history first, then sync its digest in the reservation. A normal +// process exit or successful unsynced rename alone is not a clean seal. On an +// error the caller must not assume cleanliness; restart validates whichever +// durable record survived. The seal never relaxes the next run's leader barrier. +func (r *signingReservation) seal(dir string, history *alpenglow.VoteHistory, identity ed25519.PrivateKey) error { + r.halt() + if r.uncertain { + return errors.New("uncertain reservation write; retaining unclean recovery") + } + if err := alpenglow.SaveVoteHistory(dir, history, identity); err != nil { + return err + } + digest, err := alpenglow.VoteHistoryDigest(dir, history.NodePubkey) + if err != nil { + return err + } + if r.record.Generation == math.MaxUint64 { + return errors.New("vote reservation generation exhausted") + } + next := r.record + next.Generation++ + next.CleanHistoryDigest = digest + return r.persist(next) +} + +// Retain only events blocked on renewal, not historical catch-up traffic. +// Replay the original event after acknowledgement so normal finality, parent, +// execution and invalidation checks still decide whether to vote. +func (v *alpenglowVoter) retainReservationEvent(event voterEvent) { + r := v.reservation + if r == nil { + return + } + var slot uint64 + switch event.kind { + case voterEventBlock: + slot = event.block.Block.Slot + case voterEventBlockTimeout, voterEventCrashedLeaderTimeout: + slot = event.slot | (alpenglow.LeaderWindowSlots - 1) + case voterEventConsensus: + switch event.consensus.Kind { + case alpenglow.ConsensusEventBlockNotarized, alpenglow.ConsensusEventParentReady, alpenglow.ConsensusEventSafeToNotar, alpenglow.ConsensusEventSafeToSkip: + slot = event.consensus.Slot + if event.consensus.Kind == alpenglow.ConsensusEventSafeToNotar || event.consensus.Kind == alpenglow.ConsensusEventSafeToSkip { + slot |= alpenglow.LeaderWindowSlots - 1 + } + default: + return + } + default: + return + } + if slot <= r.through.Load() || slot <= v.admissionFloor() || slot < v.waitToVoteSlot || v.engine.alpenglowVerifiedFinalityFloor() < r.recoverThrough || slot <= r.recoverThrough { + return + } + if !v.votingStarted && v.readyToVote != nil && !v.readyToVote(slot) { + return + } + r.request(slot) + if len(v.reservationEvents) < votorEventQueueSize { + v.reservationEvents = append(v.reservationEvents, event) + } +} + +// AlpenglowCanSignLeaderSlot protects every produced slot, including the +// trailing slots of a leader window. A clean vote-history marker is not a +// complete leader-block history, so leaders always skip the old reservation. +func (e *AlpenglowObserverEngine) AlpenglowCanSignLeaderSlot(slot uint64) bool { + if e.safetyError() != nil { + return false + } + e.voterMu.RLock() + defer e.voterMu.RUnlock() + v := e.voter + if v == nil { + return false + } + if v.reservation == nil { + return true + } + return v.reservation.allow(slot, e.alpenglowVerifiedFinalityFloor(), true) +} diff --git a/pkg/consensus/vote_reservation_test.go b/pkg/consensus/vote_reservation_test.go new file mode 100644 index 000000000..f4e5c418d --- /dev/null +++ b/pkg/consensus/vote_reservation_test.go @@ -0,0 +1,330 @@ +package consensus + +import ( + "crypto/ed25519" + "errors" + "math" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func reservedTestConfig(dir string) VotingConfig { + return VotingConfig{Identity: voterTestKey(11), AuthorizedVoter: voterTestKey(12), VoteAccount: solana.PublicKey(voterTestKey(13).Public().(ed25519.PublicKey)), HistoryDir: dir, Genesis: solana.Hash{1}, ReservedHistory: true, InitializeVoteReservation: true, WaitToVoteSlot: 40, ReadyToVote: func(uint64) bool { return true }, EpochForSlot: func(uint64) uint64 { return 7 }, Peers: func([]alpenglow.ValidatorStake) []alpenglow.VotorPeer { return nil }} +} + +func openReservedTestVoter(t *testing.T, cfg VotingConfig, root uint64) (*alpenglowVoter, error) { + t.Helper() + e, err := NewEngine(Config{AlpenglowIdentity: cfg.Identity, AlpenglowShredVersion: 0x1234}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, e.Close()) }) + set := voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount) + e.SetAlpenglowEpochLookup(cfg.EpochForSlot) + require.NoError(t, e.SetAlpenglowValidatorSet(set)) + block := alpenglow.BlockID{Slot: root, Hash: solana.Hash{byte(root)}} + e.SetAlpenglowRoot(block) + v, err := newAlpenglowVoterUnstarted(e, cfg, block, []alpenglow.ValidatorSet{set}) + if err == nil { + t.Cleanup(func() { require.NoError(t, v.close()) }) + } + return v, err +} + +func reserveThrough(t *testing.T, r *signingReservation, slot uint64) { + t.Helper() + r.request(slot) + require.Eventually(t, func() bool { return r.through.Load() >= slot }, time.Second, time.Millisecond) +} + +// Simulate loss of this process without executing the clean shutdown protocol. +func crashReservedTestVoter(t *testing.T, v *alpenglowVoter) { + t.Helper() + v.shutdownOnce.Do(func() { + v.closeOnce.Do(func() { close(v.done) }) + v.wg.Wait() + v.reservation.halt() + if v.historyWriter != nil { + require.NoError(t, v.historyWriter.close()) + } + require.NoError(t, v.broadcaster.Close()) + require.NoError(t, v.historyLock.Close()) + }) +} + +func TestReservedVotingLostHistorySuffixAndRepeatedCrash(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 60) + baseline, err := os.ReadFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node)) + require.NoError(t, err) + voted, err := v.cast(alpenglow.NewNotarizationVote(60, solana.Hash{1}), false) + require.NoError(t, err) + require.True(t, voted) + oldH := v.reservation.through.Load() + crashReservedTestVoter(t, v) + // Reproduce a host crash retaining a valid older version of detailed history. + require.NoError(t, os.WriteFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node), baseline, 0600)) + cfg.InitializeVoteReservation = false + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Equal(t, oldH, resumed.reservation.recoverThrough) + for _, vote := range []alpenglow.Vote{alpenglow.NewNotarizationVote(60, solana.Hash{2}), alpenglow.NewSkipVote(60), alpenglow.NewFinalizationVote(60), alpenglow.NewNotarizationFallbackVote(60, solana.Hash{2}), alpenglow.NewSkipFallbackVote(60), alpenglow.NewSkipVote(oldH + 1)} { + // Check at the signing boundary, including the restoration bypass. + for _, normal := range []bool{false, true} { + _, _, err := resumed.sign(vote, normal) + require.ErrorIs(t, err, errVoterNotReady) + } + } + require.Equal(t, oldH, resumed.reservation.through.Load(), "recovery must not keep moving its target") + crashReservedTestVoter(t, resumed) + resumed, err = openReservedTestVoter(t, cfg, oldH) + require.NoError(t, err) + require.Equal(t, oldH, resumed.reservation.recoverThrough) + reserveThrough(t, resumed.reservation, oldH+1) + voted, err = resumed.cast(alpenglow.NewSkipVote(oldH+1), false) + require.NoError(t, err) + require.True(t, voted) + voted, err = resumed.cast(alpenglow.NewSkipVote(oldH), false) + require.NoError(t, err) + require.False(t, voted) +} + +func TestReservedVotingCleanMarkerConsumedBeforeSigning(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + voted, err := v.cast(alpenglow.NewSkipVote(44), false) + require.NoError(t, err) + require.True(t, voted) + require.NoError(t, v.close()) + r, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.NotEmpty(t, r.CleanHistoryDigest) + cfg.InitializeVoteReservation = false + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Zero(t, resumed.reservation.recoverThrough) + require.True(t, resumed.history.HasSkipped(44)) + r, err = alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) + voted, err = resumed.cast(alpenglow.NewSkipVote(45), false) + require.NoError(t, err) + require.True(t, voted) + require.False(t, resumed.reservation.allow(45, 39, true), "clean vote history does not authorize repeating leader blocks") + crashReservedTestVoter(t, resumed) + again, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Equal(t, r.Through, again.reservation.recoverThrough) + // A shutdown before recovering the uncertain range must not mark it clean. + require.NoError(t, again.close()) + r, err = alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) +} + +func TestReservedVotingRejectsMissingCorruptOrWrongDomain(t *testing.T) { + for _, which := range []string{"missing_history", "missing_bound", "corrupt_bound", "genesis", "authorized", "vote_account", "synchronous"} { + t.Run(which, func(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + crashReservedTestVoter(t, v) + cfg.InitializeVoteReservation = false + switch which { + case "missing_history": + require.NoError(t, os.Remove(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node))) + case "missing_bound": + require.NoError(t, os.Remove(alpenglow.VoteReservationFilename(cfg.HistoryDir, v.node))) + cfg.InitializeVoteReservation = true + case "corrupt_bound": + require.NoError(t, os.WriteFile(alpenglow.VoteReservationFilename(cfg.HistoryDir, v.node), []byte("{"), 0600)) + case "genesis": + cfg.Genesis = solana.Hash{2} + case "authorized": + cfg.AuthorizedVoter = voterTestKey(19) + case "vote_account": + cfg.VoteAccount = solana.PublicKey{9} + case "synchronous": + cfg.ReservedHistory = false + } + _, err = openReservedTestVoter(t, cfg, 39) + require.Error(t, err) + }) + } +} + +func TestReservedVotingRequiresEnrollmentAndExclusiveOwner(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + cfg.InitializeVoteReservation = false + _, err := openReservedTestVoter(t, cfg, 39) + require.Error(t, err) + cfg.InitializeVoteReservation = true + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + _, err = openReservedTestVoter(t, cfg, 39) + require.ErrorContains(t, err, "already owned") + require.NoError(t, v.close()) +} + +func TestSigningReservationUnacknowledgedSyncCannotAuthorize(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + r := &signingReservation{record: alpenglow.VoteReservation{Through: 64, Generation: 1}, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + r.through.Store(64) + var calls atomic.Uint64 + r.persist = func(next alpenglow.VoteReservation) error { + calls.Add(1) + close(entered) + <-release + return nil + } + go r.run() + require.True(t, r.allow(64, 64, false)) + <-entered + require.Equal(t, uint64(64), r.through.Load()) + require.False(t, r.allow(65, 64, false)) + close(release) + require.Eventually(t, func() bool { return r.through.Load() > 64 }, time.Second, time.Millisecond) + require.True(t, r.allow(65, 64, false)) + r.halt() + require.Equal(t, uint64(1), calls.Load()) +} + +func TestSigningReservationUncertainWriteSurvivesRestart(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + // Stop the original worker, then exercise a fresh worker against the real record. + v.reservation.halt() + record, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + r := &signingReservation{record: record, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + r.through.Store(record.Through) + wrote := make(chan struct{}, 1) + r.persist = func(next alpenglow.VoteReservation) error { + err := alpenglow.SaveVoteReservation(cfg.HistoryDir, next, cfg.Identity) + select { + case wrote <- struct{}{}: + default: + } + if err != nil { + return err + } + return errors.New("injected lost sync acknowledgement") + } + v.reservation = r + go r.run() + require.False(t, r.allow(60, 39, false)) + <-wrote + r.halt() + require.Equal(t, record.Through, r.through.Load()) + durable, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Greater(t, durable.Through, record.Through) + require.ErrorContains(t, r.seal(cfg.HistoryDir, v.history, cfg.Identity), "uncertain") + crashReservedTestVoter(t, v) + cfg.InitializeVoteReservation = false + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Equal(t, durable.Through, resumed.reservation.recoverThrough) +} + +func TestSigningReservationNeverWraps(t *testing.T) { + r := &signingReservation{record: alpenglow.VoteReservation{Through: 64, Generation: 1}, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), persist: func(alpenglow.VoteReservation) error { t.Error("overflow attempted persistence"); return nil }} + r.through.Store(64) + go r.run() + require.False(t, r.allow(math.MaxUint64, 64, false)) + r.halt() + require.Equal(t, uint64(64), r.through.Load()) +} + +func TestReservationRetryRechecksFinality(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + event := voterEvent{kind: voterEventBlockTimeout, slot: 44} + require.NoError(t, v.handle(event)) + require.NotEmpty(t, v.reservationEvents) + reserveThrough(t, v.reservation, 44) + v.engine.SetAlpenglowRoot(alpenglow.BlockID{Slot: 47, Hash: solana.Hash{47}}) + pending := v.reservationEvents + v.reservationEvents = nil + for _, e := range pending { + require.NoError(t, v.handle(e)) + } + require.False(t, v.history.HasSkipped(44), "finalized work must not be signed after a delayed ack") +} + +func TestReservedVotingEverySignatureTypeAtBound(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + h := v.reservation.through.Load() + // Freeze acknowledgement while leaving the signing guard active. + v.reservation.halt() + v.reservation.stopped.Store(false) + defer v.reservation.stopped.Store(true) + for _, slot := range []uint64{h, h + 1} { + votes := []alpenglow.Vote{alpenglow.NewNotarizationVote(slot, solana.Hash{2}), alpenglow.NewSkipVote(slot), alpenglow.NewFinalizationVote(slot), alpenglow.NewNotarizationFallbackVote(slot, solana.Hash{2}), alpenglow.NewSkipFallbackVote(slot)} + for _, vote := range votes { + for _, normal := range []bool{false, true} { + _, _, err := v.sign(vote, normal) + if slot == h { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, errVoterNotReady) + } + } + } + } +} + +func TestReservedCleanDigestMismatchUsesCrashRecovery(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + baseline, err := os.ReadFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node)) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + voted, err := v.cast(alpenglow.NewSkipVote(44), false) + require.NoError(t, err) + require.True(t, voted) + require.NoError(t, v.close()) + require.NoError(t, os.WriteFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node), baseline, 0600)) + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.NotZero(t, resumed.reservation.recoverThrough) +} + +func TestReservationLoopRetriesAfterAcknowledgement(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + v.start() + require.NoError(t, v.enqueue(voterEvent{kind: voterEventBlockTimeout, slot: 44})) + // Read via stats, not mutable voter history, while its loop is running. + require.Eventually(t, func() bool { v.landingMu.RLock(); defer v.landingMu.RUnlock(); return v.stats.VotesCastThisRun == 4 }, time.Second, time.Millisecond) + require.NoError(t, v.close()) + for slot := uint64(44); slot <= 47; slot++ { + require.True(t, v.history.HasSkipped(slot)) + } +} + +func TestReservationRetainsWindowCrossingBound(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + h := v.reservation.through.Load() // 76: window ends at 79. + v.retainReservationEvent(voterEvent{kind: voterEventBlockTimeout, slot: h}) + require.Len(t, v.reservationEvents, 1, "trailing skip slots require retry even if the first slot fits") +} diff --git a/pkg/consensus/voter.go b/pkg/consensus/voter.go index b77a7689b..4a9cbbc15 100644 --- a/pkg/consensus/voter.go +++ b/pkg/consensus/voter.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "os" "sort" "sync" "time" @@ -38,43 +39,57 @@ type VotingPeerSource func(validators []alpenglow.ValidatorStake) []alpenglow.Vo // account address; AuthorizedVoter is the Ed25519 signer from which its BLS key // was registered. type VotingConfig struct { - Identity ed25519.PrivateKey - AuthorizedVoter ed25519.PrivateKey - VoteAccount solana.PublicKey - HistoryDir string - EpochForSlot func(slot uint64) uint64 - Peers VotingPeerSource - SlotDuration time.Duration - WaitToVoteSlot uint64 - ReadyToVote func(slot uint64) bool + Identity ed25519.PrivateKey + AuthorizedVoter ed25519.PrivateKey + VoteAccount solana.PublicKey + HistoryDir string + ReservedHistory bool + InitializeVoteReservation bool + Genesis solana.Hash + EpochForSlot func(slot uint64) uint64 + Peers VotingPeerSource + SlotDuration time.Duration + WaitToVoteSlot uint64 // Inclusive minimum for new votes; authenticated history restoration is separate + ReadyToVote func(slot uint64) bool } // VotingStats exposes positive network evidence separately from local casting. // NetworkLandedVotes counts unique persisted votes whose rank appeared in the // exact BLS-verified certificate proof received over Votor QUIC. type VotingStats struct { - Enabled bool `json:"enabled"` - VotesCastThisRun uint64 `json:"votes_cast_this_run"` - NetworkLandedVotes uint64 `json:"network_landed_votes"` - LastNetworkLandedSlot uint64 `json:"last_network_landed_slot,omitempty"` - LastNetworkLandedVoteType alpenglow.VoteType `json:"last_network_landed_vote_type,omitempty"` - LastNetworkCertificateType alpenglow.CertificateType `json:"last_network_certificate_type,omitempty"` - LastNetworkLandedAt time.Time `json:"last_network_landed_at,omitempty"` - BroadcastMessagesQueued uint64 `json:"broadcast_messages_queued"` - BroadcastMessagesDropped uint64 `json:"broadcast_messages_dropped"` - BroadcastPeerSends uint64 `json:"broadcast_peer_sends"` - BroadcastPeerSendsSkipped uint64 `json:"broadcast_peer_sends_skipped"` - BroadcastPeerSendErrors uint64 `json:"broadcast_peer_send_errors"` - BroadcastDesiredPeers int `json:"broadcast_desired_peers"` - BroadcastActiveConnections int `json:"broadcast_active_connections"` - BroadcastPendingConnections int `json:"broadcast_pending_connections"` - BroadcastConnectionAttempts uint64 `json:"broadcast_connection_attempts"` - BroadcastConnectionErrors uint64 `json:"broadcast_connection_errors"` - BroadcastConnectionJobsDropped uint64 `json:"broadcast_connection_jobs_dropped"` - BroadcastLastPeerSendError string `json:"broadcast_last_peer_send_error,omitempty"` - BroadcastLastPeerSendErrorAt time.Time `json:"broadcast_last_peer_send_error_at,omitempty"` - BroadcastLastConnectionError string `json:"broadcast_last_connection_error,omitempty"` - BroadcastLastConnectionErrorAt time.Time `json:"broadcast_last_connection_error_at,omitempty"` + HistorySnapshotsSubmitted uint64 `json:"history_snapshots_submitted,omitempty"` + HistorySnapshotsWritten uint64 `json:"history_snapshots_written,omitempty"` + HistorySnapshotsCoalesced uint64 `json:"history_snapshots_coalesced,omitempty"` + ReservedHistory bool `json:"reserved_history"` + SigningReservedThrough uint64 `json:"signing_reserved_through,omitempty"` + RecoveryThrough uint64 `json:"recovery_through,omitempty"` + Enabled bool `json:"enabled"` + VotesCastThisRun uint64 `json:"votes_cast_this_run"` + NetworkLandedVotes uint64 `json:"network_landed_votes"` + LastNetworkLandedSlot uint64 `json:"last_network_landed_slot,omitempty"` + LastNetworkLandedVoteType alpenglow.VoteType `json:"last_network_landed_vote_type,omitempty"` + LastNetworkCertificateType alpenglow.CertificateType `json:"last_network_certificate_type,omitempty"` + LastNetworkLandedAt time.Time `json:"last_network_landed_at,omitempty"` + BroadcastMessagesQueued uint64 `json:"broadcast_messages_queued"` + BroadcastMessagesDropped uint64 `json:"broadcast_messages_dropped"` + BroadcastPeerSends uint64 `json:"broadcast_peer_sends"` + BroadcastPeerSendsSkipped uint64 `json:"broadcast_peer_sends_skipped"` + BroadcastPeerSendErrors uint64 `json:"broadcast_peer_send_errors"` + BroadcastPeerQueueDrops uint64 `json:"broadcast_peer_queue_drops"` + BroadcastPeerQueueDiscarded uint64 `json:"broadcast_peer_queue_discarded"` + BroadcastPeerSendTimeouts uint64 `json:"broadcast_peer_send_timeouts"` + BroadcastPeerQueueMaxDelay time.Duration `json:"broadcast_peer_queue_max_delay_ns"` + BroadcastPeerQueues []alpenglow.VotorPeerQueueStats `json:"broadcast_peer_queues,omitempty"` + BroadcastDesiredPeers int `json:"broadcast_desired_peers"` + BroadcastActiveConnections int `json:"broadcast_active_connections"` + BroadcastPendingConnections int `json:"broadcast_pending_connections"` + BroadcastConnectionAttempts uint64 `json:"broadcast_connection_attempts"` + BroadcastConnectionErrors uint64 `json:"broadcast_connection_errors"` + BroadcastConnectionJobsDropped uint64 `json:"broadcast_connection_jobs_dropped"` + BroadcastLastPeerSendError string `json:"broadcast_last_peer_send_error,omitempty"` + BroadcastLastPeerSendErrorAt time.Time `json:"broadcast_last_peer_send_error_at,omitempty"` + BroadcastLastConnectionError string `json:"broadcast_last_connection_error,omitempty"` + BroadcastLastConnectionErrorAt time.Time `json:"broadcast_last_connection_error_at,omitempty"` } type voterEventKind uint8 @@ -88,6 +103,7 @@ const ( voterEventValidatorSet voterEventRoot voterEventNetworkCertificate + voterEventDurableRoot ) type voterEvent struct { @@ -109,43 +125,49 @@ type pendingVotorBlock struct { // history decisions run on loop; validator-set snapshots are protected only so // the outbound peer callback can read them from broadcast workers. type alpenglowVoter struct { - engine *AlpenglowObserverEngine - identity ed25519.PrivateKey - node solana.PublicKey - voteAccount solana.PublicKey - signer *alpenglow.BLSSigner - historyDir string - history *alpenglow.VoteHistory - epochForSlot func(uint64) uint64 - peerSource VotingPeerSource - slotDuration time.Duration - waitToVoteSlot uint64 - readyToVote func(slot uint64) bool - broadcaster *alpenglow.VotorBroadcaster - events chan voterEvent - done chan struct{} - startOnce sync.Once - closeOnce sync.Once - wg sync.WaitGroup - setsMu sync.RWMutex - sets map[uint64]alpenglow.ValidatorSet - restored map[alpenglow.VoteMessageKey]bool - pending map[uint64][]pendingVotorBlock - receivedShred map[uint64]bool - timeoutsSet map[uint64]bool - executedBlocks map[alpenglow.BlockID]bool - highestFinal uint64 - lastFinalizedAt time.Time - votingStarted bool - latestLiveSlot uint64 - standstillSlot *uint64 - refreshQueue []alpenglow.Message - refreshCursor int - lastWarn map[uint64]time.Time - landingMu sync.RWMutex - landed map[alpenglow.VoteMessageKey]struct{} - stats VotingStats - lastStatsLog time.Time + engine *AlpenglowObserverEngine + identity ed25519.PrivateKey + node solana.PublicKey + voteAccount solana.PublicKey + signer *alpenglow.BLSSigner + historyDir string + historyLock *os.File + reservation *signingReservation + historyWriter *voteHistoryWriter + reservationEvents []voterEvent + shutdownOnce sync.Once + shutdownErr error + history *alpenglow.VoteHistory + epochForSlot func(uint64) uint64 + peerSource VotingPeerSource + slotDuration time.Duration + waitToVoteSlot uint64 + readyToVote func(slot uint64) bool + broadcaster *alpenglow.VotorBroadcaster + events chan voterEvent + done chan struct{} + startOnce sync.Once + closeOnce sync.Once + wg sync.WaitGroup + setsMu sync.RWMutex + sets map[uint64]alpenglow.ValidatorSet + restored map[alpenglow.VoteMessageKey]bool + pending map[uint64][]pendingVotorBlock + receivedShred map[uint64]bool + timeoutsSet map[uint64]bool + executedBlocks map[alpenglow.BlockID]bool + highestFinal uint64 + lastFinalizedAt time.Time + votingStarted bool + latestLiveSlot uint64 + standstillSlot *uint64 + refreshQueue []alpenglow.Message + refreshCursor int + lastWarn map[uint64]time.Time + landingMu sync.RWMutex + landed map[alpenglow.VoteMessageKey]struct{} + stats VotingStats + lastStatsLog time.Time // beforeVoteGuard is a deterministic test seam for invalidation races. It // is nil in production. beforeVoteGuard func(alpenglow.BlockID) @@ -163,6 +185,9 @@ func newAlpenglowVoterUnstarted(engine *AlpenglowObserverEngine, cfg VotingConfi } func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfig, root alpenglow.BlockID, start bool, sets []alpenglow.ValidatorSet) (*alpenglowVoter, error) { + if cfg.InitializeVoteReservation && !cfg.ReservedHistory { + return nil, errors.New("initialize-vote-reservation requires reserved-vote-history") + } if engine == nil { return nil, fmt.Errorf("enable Alpenglow voting: nil consensus engine") } @@ -192,17 +217,52 @@ func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfi if node != engineNode { return nil, fmt.Errorf("enable Alpenglow voting: identity %s does not match consensus transport identity %s", node, engineNode) } + historyLock, err := alpenglow.LockVoteHistory(cfg.HistoryDir, node) + if err != nil { + return nil, err + } + keepLock := false + defer func() { + if !keepLock { + historyLock.Close() + } + }() + if !cfg.ReservedHistory { + if _, err := os.Stat(alpenglow.VoteReservationFilename(cfg.HistoryDir, node)); !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("existing or unreadable vote reservation requires reserved history mode") + } + } history, err := alpenglow.LoadVoteHistory(cfg.HistoryDir, node) if err != nil { if !errors.Is(err, alpenglow.ErrVoteHistoryNotFound) { return nil, fmt.Errorf("enable Alpenglow voting: refuse unsafe vote-history reset: %w", err) } + if cfg.ReservedHistory { + if _, err := os.Stat(alpenglow.VoteReservationFilename(cfg.HistoryDir, node)); !errors.Is(err, os.ErrNotExist) || !cfg.InitializeVoteReservation { + return nil, fmt.Errorf("missing history for reserved voter; refusing automatic reset") + } + } history = alpenglow.NewVoteHistory(node, root.Slot) if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { return nil, fmt.Errorf("initialize Alpenglow vote history: %w", err) } mlog.Log.FileOnlyf("ALPENGLOW voting: created new vote history for %s at root %d; do not reuse this vote account on another validator", node, root.Slot) } + if history.ReservationRequired && !cfg.ReservedHistory { + return nil, errors.New("reserved vote history cannot be opened in synchronous mode") + } + var reservation *signingReservation + if cfg.ReservedHistory { + reservation, err = openSigningReservation(cfg, node, engine.shredVersion, history) + if err != nil { + return nil, err + } + defer func() { + if !keepLock { + reservation.halt() + } + }() + } if history.Root < root.Slot { history.SetRoot(root.Slot) if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { @@ -220,6 +280,8 @@ func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfi voteAccount: cfg.VoteAccount, signer: signer, historyDir: cfg.HistoryDir, + historyLock: historyLock, + reservation: reservation, history: history, epochForSlot: cfg.EpochForSlot, peerSource: cfg.Peers, @@ -261,6 +323,12 @@ func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfi return nil, err } v.broadcaster = broadcaster + if reservation != nil { + v.historyWriter = newVoteHistoryWriter(func(snapshot *alpenglow.VoteHistorySnapshot) error { + return alpenglow.SaveReservedVoteHistorySnapshot(v.historyDir, snapshot) + }, v.failHistoryWrite) + } + keepLock = true if start { v.start() } @@ -300,12 +368,25 @@ func (v *alpenglowVoter) loop() { v.closeOnce.Do(func() { close(v.done) }) v.wg.Done() }() + var reservationChanged <-chan struct{} + if v.reservation != nil { + reservationChanged = v.reservation.changed + } ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-v.done: return + case <-reservationChanged: + pending := v.reservationEvents + v.reservationEvents = nil + for _, event := range pending { + if err := v.handle(event); err != nil { + v.engine.latchSafetyError(fmt.Errorf("reservation retry: %w", err)) + return + } + } case event := <-v.events: if err := v.handle(event); err != nil { v.engine.latchSafetyError(fmt.Errorf("voting engine: %w", err)) @@ -324,6 +405,11 @@ func (v *alpenglowVoter) loop() { } func (v *alpenglowVoter) handle(event voterEvent) error { + v.retainReservationEvent(event) + return v.handleEvent(event) +} + +func (v *alpenglowVoter) handleEvent(event voterEvent) error { floor := v.admissionFloor() if event.kind == voterEventBlock && v.engine.ensureChain().IsObjectivelyInvalidBlock(event.block.Block) { return nil @@ -363,6 +449,9 @@ func (v *alpenglowVoter) handle(event voterEvent) error { return nil } return v.saveHistory() + case voterEventDurableRoot: + root := v.engine.applyAlpenglowDurableRoot(event.slot) + return v.handleEvent(voterEvent{kind: voterEventRoot, root: root}) case voterEventNetworkCertificate: v.recordNetworkCertificate(event.certificate) return nil @@ -477,9 +566,9 @@ func (v *alpenglowVoter) handleConsensus(event alpenglow.ConsensusEvent) error { func (v *alpenglowVoter) admissionFloor() uint64 { floor := v.history.Root - if v.highestFinal > floor { - floor = v.highestFinal - } + // highestFinal tracks network progress and standstill, not retirement of + // our own decisions. Keep ParentReady, pending replay and exact vote history + // available until the retained pool or an ordered durable root retires them. if engineFloor := v.engine.alpenglowVoteActionFloor(); engineFloor > floor { floor = engineFloor } @@ -661,7 +750,7 @@ func (v *alpenglowVoter) castTarget(vote alpenglow.Vote, restoring bool, guarded if !restoring && v.beforeVoteGuard != nil { v.beforeVoteGuard(guardedBlock) } - // Hold through signing, durable history, pool admission, and broadcast. + // Hold through signing, history recording, pool admission, and broadcast. // Objective invalidation takes the write side before changing the chain, // so a new or restored vote is wholly before it or sees the tombstone. v.engine.invalidActionMu.RLock() @@ -682,18 +771,22 @@ func (v *alpenglowVoter) castTarget(vote alpenglow.Vote, restoring bool, guarded return false, nil } if !restoring { - // Finality can advance while the BLS signature is computed. Avoid a - // durable stale record when that race is already visible here; if it - // advances later, atomic admission below classifies it benignly. + // Retention/root pruning can advance while the BLS signature is computed. + // Avoid an expired record if that race is already visible here; atomic + // admission below classifies a later pruning race benignly. if vote.Slot <= v.admissionFloor() { return false, nil } if err := v.history.AddVote(vote); err != nil { return false, fmt.Errorf("record %s vote at slot %d: %w", vote.Type, vote.Slot, err) } - // Pool admission may synchronously assemble and publish a certificate. - // Persist the anti-equivocation record first so no externally visible - // proof can survive a crash without its signed local history. + // Pool admission can publish a certificate. In reserved mode the durable + // upper bound covers loss of this unsynchronized history replacement; + // synchronous mode still persists the exact history before admission. + // sign computed BLS bytes in RAM, but nothing may expose them before + // this boundary succeeds. In reserved mode saveHistory only queues the + // snapshot: restart safety comes from the durable reservation checked + // before sign, not from assuming this snapshot reached durable storage. if err := v.saveHistory(); err != nil { return false, err } @@ -727,7 +820,16 @@ func (v *alpenglowVoter) castTarget(vote alpenglow.Vote, restoring bool, guarded return true, nil } +// sign checks reservation recovery even when restoration bypasses the live +// joining gate. Re-signing a saved vote is still signing; the presence of an +// older valid history file cannot prove that its lost suffix was conflict-free. func (v *alpenglowVoter) sign(vote alpenglow.Vote, respectVotingGate bool) (alpenglow.VoteMessage, alpenglow.VoteVerifyResult, error) { + if err := v.engine.safetyError(); err != nil { + return alpenglow.VoteMessage{}, alpenglow.VoteVerifyResult{}, err + } + if v.reservation != nil && !v.reservation.allow(vote.Slot, v.engine.alpenglowVerifiedFinalityFloor(), false) { + return alpenglow.VoteMessage{}, alpenglow.VoteVerifyResult{}, fmt.Errorf("%w: waiting for verified recovery or durable signing reservation", errVoterNotReady) + } if respectVotingGate { if err := v.votingGateError(vote.Slot); err != nil { return alpenglow.VoteMessage{}, alpenglow.VoteVerifyResult{}, err @@ -768,7 +870,7 @@ func (v *alpenglowVoter) votingGateError(slot uint64) error { return fmt.Errorf("%w: slot %d is at or below consensus action floor %d", errVoterNotReady, slot, floor) } if slot < v.waitToVoteSlot { - return fmt.Errorf("%w: waiting for startup watermark slot %d", errVoterNotReady, v.waitToVoteSlot) + return fmt.Errorf("%w: waiting for voting cutoff slot %d", errVoterNotReady, v.waitToVoteSlot) } // ReadyToVote is a startup join guard, not a perpetual clock check. Once an // accepted live block or vote joins Votor, verified ParentReady and timeout @@ -777,6 +879,9 @@ func (v *alpenglowVoter) votingGateError(slot uint64) error { if !v.votingStarted && v.readyToVote != nil && !v.readyToVote(slot) { return fmt.Errorf("%w: slot is still behind the startup live voting window", errVoterNotReady) } + if v.reservation != nil && !v.reservation.allow(slot, v.engine.alpenglowVerifiedFinalityFloor(), false) { + return fmt.Errorf("%w: waiting for verified recovery or durable signing reservation", errVoterNotReady) + } return nil } @@ -837,6 +942,9 @@ func (v *alpenglowVoter) votorTransportValidators() []alpenglow.ValidatorStake { } func (v *alpenglowVoter) restoreVotesForEpoch(epoch uint64) error { + if v.reservation != nil && v.reservation.recoverThrough != 0 { + return nil + } floor := v.admissionFloor() for _, vote := range v.history.VotesAfter(v.history.Root - minU64(v.history.Root, 1)) { // Keep every signed history entry for anti-equivocation, but do not @@ -884,6 +992,9 @@ func (v *alpenglowVoter) restoreVotesForEpoch(epoch uint64) error { // persist-before-admission crash window without trusting process-local invalid // block state across a restart. func (v *alpenglowVoter) restoreVotesForBlock(block alpenglow.BlockID) (bool, error) { + if v.reservation != nil && v.reservation.recoverThrough != 0 { + return false, nil + } if block.Slot <= v.admissionFloor() { return false, nil } @@ -1086,13 +1197,29 @@ func (v *alpenglowVoter) isIdentityStaked(slot uint64) bool { return false } +// saveHistory is a publication boundary with different persistence semantics: +// synchronous mode acknowledges durable exact history; reserved mode acknowledges +// an immutable queued snapshot only. The latter relies on the signing reservation. func (v *alpenglowVoter) saveHistory() error { + if v.historyWriter != nil { + snapshot, err := alpenglow.PrepareReservedVoteHistory(v.history, v.identity) + if err != nil { + return err + } + return v.historyWriter.submit(snapshot) + } if err := alpenglow.SaveVoteHistory(v.historyDir, v.history, v.identity); err != nil { return fmt.Errorf("persist vote history before consensus publication: %w", err) } return nil } +func (v *alpenglowVoter) failHistoryWrite(err error) { + v.engine.latchSafetyError(err) + mlog.Log.Errorf("ALPENGLOW VOTING SAFETY: %v", err) + v.closeOnce.Do(func() { close(v.done) }) +} + func (v *alpenglowVoter) recordNetworkCertificate(cert alpenglow.Certificate) { set, ok := v.validatorSet(cert.Slot) if !ok { @@ -1201,6 +1328,14 @@ func (v *alpenglowVoter) snapshot() VotingStats { stats := v.stats v.landingMu.RUnlock() stats.Enabled = true + if v.historyWriter != nil { + stats.HistorySnapshotsSubmitted, stats.HistorySnapshotsWritten, stats.HistorySnapshotsCoalesced = v.historyWriter.counters() + } + if v.reservation != nil { + stats.ReservedHistory = true + stats.SigningReservedThrough = v.reservation.through.Load() + stats.RecoveryThrough = v.reservation.recoverThrough + } if v.broadcaster != nil { broadcast := v.broadcaster.Stats() stats.BroadcastMessagesQueued = broadcast.MessagesQueued @@ -1208,6 +1343,11 @@ func (v *alpenglowVoter) snapshot() VotingStats { stats.BroadcastPeerSends = broadcast.PeerSends stats.BroadcastPeerSendsSkipped = broadcast.PeerSendsSkipped stats.BroadcastPeerSendErrors = broadcast.PeerSendErrors + stats.BroadcastPeerQueueDrops = broadcast.PeerQueueDrops + stats.BroadcastPeerQueueDiscarded = broadcast.PeerQueueDiscarded + stats.BroadcastPeerSendTimeouts = broadcast.PeerSendTimeouts + stats.BroadcastPeerQueueMaxDelay = broadcast.PeerQueueMaxDelay + stats.BroadcastPeerQueues = broadcast.PeerQueues stats.BroadcastDesiredPeers = broadcast.DesiredPeers stats.BroadcastActiveConnections = broadcast.Connections stats.BroadcastPendingConnections = broadcast.PendingConnections @@ -1228,7 +1368,7 @@ func (v *alpenglowVoter) maybeLogStats() { } v.lastStatsLog = time.Now() stats := v.snapshot() - mlog.Log.FileOnlyf("alpenglow voting stats: votes_cast_this_run=%d network_landed=%d last_landed_slot=%d broadcast_queued=%d broadcast_dropped=%d peer_sends=%d peer_sends_skipped=%d peer_send_errors=%d desired_peers=%d active_connections=%d pending_connections=%d connection_attempts=%d connection_errors=%d connection_jobs_dropped=%d", + mlog.Log.FileOnlyf("alpenglow voting stats: votes_cast_this_run=%d network_landed=%d last_landed_slot=%d broadcast_queued=%d broadcast_dropped=%d peer_sends=%d peer_sends_skipped=%d peer_send_errors=%d peer_queue_drops=%d peer_queue_discarded=%d peer_send_timeouts=%d peer_queue_max_delay=%s desired_peers=%d active_connections=%d pending_connections=%d connection_attempts=%d connection_errors=%d connection_jobs_dropped=%d reserved_history=%t signing_through=%d recovery_through=%d history_submitted=%d history_written=%d history_coalesced=%d", stats.VotesCastThisRun, stats.NetworkLandedVotes, stats.LastNetworkLandedSlot, @@ -1237,12 +1377,18 @@ func (v *alpenglowVoter) maybeLogStats() { stats.BroadcastPeerSends, stats.BroadcastPeerSendsSkipped, stats.BroadcastPeerSendErrors, + stats.BroadcastPeerQueueDrops, + stats.BroadcastPeerQueueDiscarded, + stats.BroadcastPeerSendTimeouts, + stats.BroadcastPeerQueueMaxDelay, stats.BroadcastDesiredPeers, stats.BroadcastActiveConnections, stats.BroadcastPendingConnections, stats.BroadcastConnectionAttempts, stats.BroadcastConnectionErrors, stats.BroadcastConnectionJobsDropped, + stats.ReservedHistory, stats.SigningReservedThrough, stats.RecoveryThrough, + stats.HistorySnapshotsSubmitted, stats.HistorySnapshotsWritten, stats.HistorySnapshotsCoalesced, ) } @@ -1274,9 +1420,28 @@ func (v *alpenglowVoter) close() error { if v == nil { return nil } - v.closeOnce.Do(func() { close(v.done) }) - v.wg.Wait() - return v.broadcaster.Close() + v.shutdownOnce.Do(func() { + v.closeOnce.Do(func() { close(v.done) }) + v.wg.Wait() + if v.reservation != nil { + v.reservation.halt() + if v.historyWriter != nil { + v.shutdownErr = v.historyWriter.close() + } + // Exiting normally is insufficient: an unresolved recovery barrier, + // writer failure or safety fault must leave the session unsealed. + floor := v.engine.alpenglowVerifiedFinalityFloor() + if v.shutdownErr == nil && v.engine.safetyError() == nil && floor >= v.reservation.recoverThrough { + v.history.SetRoot(floor) + v.shutdownErr = v.reservation.seal(v.historyDir, v.history, v.identity) + if v.shutdownErr == nil { + mlog.Log.Infof("ALPENGLOW signing reservation: clean history sealed at root=%d through=%d", v.history.Root, v.reservation.through.Load()) + } + } + } + v.shutdownErr = errors.Join(v.shutdownErr, v.broadcaster.Close(), v.historyLock.Close()) + }) + return v.shutdownErr } func pendingContains(blocks []pendingVotorBlock, candidate pendingVotorBlock) bool { diff --git a/pkg/consensus/voter_finality_ordering_test.go b/pkg/consensus/voter_finality_ordering_test.go new file mode 100644 index 000000000..a53abb640 --- /dev/null +++ b/pkg/consensus/voter_finality_ordering_test.go @@ -0,0 +1,192 @@ +package consensus + +import ( + "context" + "crypto/ed25519" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Keep the actor unstarted so tests can place finality, replay and durable +// promotion in a deterministic order, using the real engine event queue. +func newOrderingTestVoter(t *testing.T, reserved bool) (*AlpenglowObserverEngine, *alpenglowVoter) { + t.Helper() + cfg := reservedTestConfig(t.TempDir()) + cfg.ReservedHistory = reserved + cfg.InitializeVoteReservation = reserved + root := alpenglow.BlockID{Slot: 39, Hash: solana.Hash{39}} + e, err := NewEngine(Config{AlpenglowIdentity: cfg.Identity, AlpenglowShredVersion: 0x1234}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, e.Close()) }) + set := voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount) + e.SetAlpenglowEpochLookup(cfg.EpochForSlot) + require.NoError(t, e.SetAlpenglowValidatorSet(set)) + e.SetAlpenglowRoot(root) + v, err := newAlpenglowVoterUnstarted(e, cfg, root, []alpenglow.ValidatorSet{set}) + require.NoError(t, err) + e.voter = v + if reserved { + reserveThrough(t, v.reservation, 44) + } + // Equivalent to startup's trusted-root ParentReady seed. + require.True(t, v.history.AddParentReady(40, root)) + return e, v +} + +func drainOrderingEvents(t *testing.T, v *alpenglowVoter) { + t.Helper() + for i := 0; i < 1000; i++ { + select { + case event := <-v.events: + require.NoError(t, v.handle(event)) + default: + return + } + } + t.Fatal("voter event queue did not drain") +} + +func observeOrderingBlock(t *testing.T, e *AlpenglowObserverEngine, slot uint64) alpenglow.BlockID { + t.Helper() + id := alpenglow.BlockID{Slot: slot, Hash: solana.Hash{byte(slot)}} + require.NoError(t, e.ObserveBlock(context.Background(), BlockObservation{Source: "ordering-test", Block: &block.Block{ + Slot: slot, ParentSlot: slot - 1, + AlpenglowBlockID: [32]byte(id.Hash), HasAlpenglowBlockID: true, + AlpenglowParentBlockID: [32]byte{byte(slot - 1)}, HasAlpenglowParentBlockID: true, + }})) + return id +} + +func finalizeOrderingBlock(t *testing.T, e *AlpenglowObserverEngine, v *alpenglowVoter, id alpenglow.BlockID) { + t.Helper() + // The two peers supply the 60% slow-finality quorum without our vote; + // adding our 30% notarization later can produce a fast certificate. + for _, vote := range []alpenglow.Vote{alpenglow.NewNotarizationVote(id.Slot, id.Hash), alpenglow.NewFinalizationVote(id.Slot)} { + for rank, key := range []ed25519.PrivateKey{voterTestKey(21), voterTestKey(22)} { + peer := signedVerifiedVoterPeerVote(t, e, v.sets[7], key, uint16(rank+1), vote) + _, err := e.acceptVerifiedVoteResult(peer) + require.NoError(t, err) + } + } + require.Equal(t, id.Slot, e.ensureChain().Snapshot().LatestDirectFinalizedBlock.Slot) +} + +func TestAlpenglowVoterReplaysFourBlocksAfterNetworkFinality(t *testing.T) { + for _, reserved := range []bool{false, true} { + name := "synchronous" + if reserved { + name = "reserved" + } + t.Run(name, func(t *testing.T) { + e, v := newOrderingTestVoter(t, reserved) + for slot := uint64(40); slot <= 43; slot++ { + id := observeOrderingBlock(t, e, slot) + finalizeOrderingBlock(t, e, v, id) + drainOrderingEvents(t, v) + require.Equal(t, slot, v.highestFinal) + require.Less(t, v.admissionFloor(), slot) + require.False(t, v.history.VotedAt(slot), "network finality does not prove local execution") + before := v.snapshot() + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: slot, Source: "ordering-test"})) + drainOrderingEvents(t, v) + hash, ok := v.history.NotarizedVote(slot) + require.True(t, ok, "replay must still notarize after slow finality") + require.Equal(t, id.Hash, hash) + require.Greater(t, v.snapshot().BroadcastMessagesQueued, before.BroadcastMessagesQueued) + require.NoError(t, e.AlpenglowSafetyError()) + } + }) + } +} + +func TestAlpenglowVoterNetworkFinalityDuringLocalAdmission(t *testing.T) { + e, v := newOrderingTestVoter(t, false) + id := observeOrderingBlock(t, e, 40) + called := false + v.beforeLocalVoteInject = func(vote alpenglow.Vote) { + if called { + return + } + called = true + require.Equal(t, alpenglow.NewNotarizationVote(id.Slot, id.Hash), vote) + finalizeOrderingBlock(t, e, v, id) + } + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: id.Slot})) + drainOrderingEvents(t, v) + require.True(t, called) + require.Positive(t, v.snapshot().VotesCastThisRun) + message, _, err := v.sign(alpenglow.NewNotarizationVote(id.Slot, id.Hash), false) + require.NoError(t, err) + require.True(t, e.ensurePool().HasVerifiedVote(message)) + require.NoError(t, e.AlpenglowSafetyError()) +} + +func TestAlpenglowVoterDurableRootCannotOvertakeQueuedReplay(t *testing.T) { + e, v := newOrderingTestVoter(t, true) + id := observeOrderingBlock(t, e, 40) + finalizeOrderingBlock(t, e, v, id) + drainOrderingEvents(t, v) + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: id.Slot})) + e.PruneAlpenglowBefore(id.Slot) + require.Equal(t, uint64(39), e.ensurePool().Snapshot().RootSlot) + require.Contains(t, e.executedReplayBlocks, id, "queued replay must retain its execution proof") + queuedBefore := v.snapshot().BroadcastMessagesQueued + drainOrderingEvents(t, v) + require.Greater(t, v.snapshot().BroadcastMessagesQueued, queuedBefore) + require.Equal(t, id.Slot, v.history.Root) + require.Equal(t, id.Slot, e.ensurePool().Snapshot().RootSlot) + require.NotContains(t, e.executedReplayBlocks, id) + _, ok := v.history.NotarizedVote(id.Slot) + require.True(t, ok, "root vote must remain available to its intra-window child") + child := observeOrderingBlock(t, e, 41) + finalizeOrderingBlock(t, e, v, child) + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: child.Slot})) + drainOrderingEvents(t, v) + hash, ok := v.history.NotarizedVote(child.Slot) + require.True(t, ok) + require.Equal(t, child.Hash, hash) + require.NoError(t, e.AlpenglowSafetyError()) +} + +func TestAlpenglowVoterFinalityPreservesEarlierSkipDecision(t *testing.T) { + e, v := newOrderingTestVoter(t, true) + require.NoError(t, v.history.AddVote(alpenglow.NewSkipVote(40))) + id := observeOrderingBlock(t, e, 40) + finalizeOrderingBlock(t, e, v, id) + drainOrderingEvents(t, v) + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: 40})) + drainOrderingEvents(t, v) + require.True(t, v.history.HasSkipped(40)) + _, ok := v.history.NotarizedVote(40) + require.False(t, ok, "late replay must never replace an earlier round-one decision") + require.NoError(t, e.AlpenglowSafetyError()) +} + +func TestReservedRecoveryUsesVerifiedFinalityNotLiveAdmissionFloor(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 40) + h := v.reservation.through.Load() + crashReservedTestVoter(t, v) + cfg.InitializeVoteReservation = false + v, err = openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + _, _, err = v.sign(alpenglow.NewSkipVote(h+1), false) + require.ErrorIs(t, err, errVoterNotReady) + id := observeOrderingBlock(t, v.engine, h) + finalizeOrderingBlock(t, v.engine, v, id) + require.Less(t, v.admissionFloor(), h) + require.Equal(t, h, v.engine.alpenglowVerifiedFinalityFloor()) + reserveThrough(t, v.reservation, h+1) + for _, normal := range []bool{false, true} { + _, _, err = v.sign(alpenglow.NewSkipVote(h), normal) + require.ErrorIs(t, err, errVoterNotReady) + _, _, err = v.sign(alpenglow.NewSkipVote(h+1), normal) + require.NoError(t, err) + } +} diff --git a/pkg/consensus/voter_wait_slot_test.go b/pkg/consensus/voter_wait_slot_test.go new file mode 100644 index 000000000..b06a35819 --- /dev/null +++ b/pkg/consensus/voter_wait_slot_test.go @@ -0,0 +1,91 @@ +package consensus + +import ( + "crypto/ed25519" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func waitSlotTestVoter(t *testing.T, cutoff uint64) *alpenglowVoter { + t.Helper() + identity, authorized := voterTestKey(11), voterTestKey(12) + voteAccount := solana.PublicKey(voterTestKey(13).Public().(ed25519.PublicKey)) + set := voterTestValidatorSet(t, identity, authorized, voteAccount) + root := alpenglow.BlockID{Slot: 39, Hash: solana.Hash{0x39}} + engine, err := NewEngine(Config{AlpenglowShredVersion: 0x1234, AlpenglowIdentity: identity}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + engine.SetAlpenglowEpochLookup(func(uint64) uint64 { return set.Epoch }) + require.NoError(t, engine.SetAlpenglowValidatorSet(set)) + engine.SetAlpenglowRoot(root) + voter, err := newAlpenglowVoterUnstarted(engine, VotingConfig{ + Identity: identity, AuthorizedVoter: authorized, VoteAccount: voteAccount, + HistoryDir: t.TempDir(), EpochForSlot: func(uint64) uint64 { return set.Epoch }, + Peers: func([]alpenglow.ValidatorStake) []alpenglow.VotorPeer { return nil }, + WaitToVoteSlot: cutoff, ReadyToVote: func(uint64) bool { return true }, + }, root, []alpenglow.ValidatorSet{set}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, voter.close()) }) + return voter +} + +func TestWaitToVoteSlotGatesEveryNewVoteType(t *testing.T) { + voter := waitSlotTestVoter(t, 44) + constructors := []func(uint64) alpenglow.Vote{ + func(slot uint64) alpenglow.Vote { return alpenglow.NewNotarizationVote(slot, solana.Hash{1}) }, + alpenglow.NewFinalizationVote, + alpenglow.NewSkipVote, + func(slot uint64) alpenglow.Vote { return alpenglow.NewNotarizationFallbackVote(slot, solana.Hash{1}) }, + alpenglow.NewSkipFallbackVote, + } + for _, started := range []bool{false, true} { + voter.votingStarted = started + for _, voteAt := range constructors { + vote := voteAt(43) + _, _, err := voter.sign(vote, true) + require.ErrorIs(t, err, errVoterNotReady, "%s started=%t", vote.Type, started) + for _, slot := range []uint64{44, 45} { + message, _, err := voter.sign(voteAt(slot), true) + require.NoError(t, err, "%s slot=%d started=%t", vote.Type, slot, started) + require.Equal(t, voteAt(slot), message.Vote) + } + } + } +} + +func TestWaitToVoteSlotSplitsSkipWindowAndPersistsOnlyAllowedVotes(t *testing.T) { + voter := waitSlotTestVoter(t, 42) + require.NoError(t, voter.trySkipWindow(40)) + for _, slot := range []uint64{40, 41} { + require.False(t, voter.history.VotedAt(slot)) + } + for _, slot := range []uint64{42, 43} { + require.True(t, voter.history.HasSkipped(slot)) + } + restored, err := alpenglow.LoadVoteHistory(voter.historyDir, voter.node) + require.NoError(t, err) + require.Equal(t, voter.history.VotesCast, restored.VotesCast) + require.EqualValues(t, 2, voter.engine.ensurePool().Snapshot().VerifiedVotes) + // Joining live voting must not make older slots eligible afterward. + require.True(t, voter.votingStarted) + voted, err := voter.cast(alpenglow.NewSkipVote(41), false) + require.NoError(t, err) + require.False(t, voted) + require.False(t, voter.history.VotedAt(41)) +} + +func TestWaitToVoteSlotPreservesAuthenticatedHistoryRestoration(t *testing.T) { + voter := waitSlotTestVoter(t, 44) + require.NoError(t, voter.history.AddVote(alpenglow.NewSkipVote(40))) + require.NoError(t, voter.saveHistory()) + restored, err := alpenglow.LoadVoteHistory(voter.historyDir, voter.node) + require.NoError(t, err) + voter.history = restored + require.NoError(t, voter.restoreVotesForEpoch(voter.epochForSlot(40))) + require.EqualValues(t, 1, voter.engine.ensurePool().Snapshot().VerifiedVotes) + require.False(t, voter.votingStarted, "restoring a recorded vote must not bypass startup readiness") + require.ErrorIs(t, voter.votingGateError(41), errVoterNotReady) +}