From fd829f61cb04dae55557471fd79c268f44a132dd Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Sun, 13 Sep 2026 23:12:41 -0500 Subject: [PATCH 01/13] Advance replay progress without waiting for certificate verification --- pkg/alpenglow/certpool.go | 22 +++-- pkg/alpenglow/certpool_progress_test.go | 112 ++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 9 deletions(-) create mode 100644 pkg/alpenglow/certpool_progress_test.go diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index 60b1882e3..50a5c5957 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -6,6 +6,7 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" blsfr "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381/fr" @@ -171,8 +172,8 @@ type CertPool struct { slots map[uint64]*poolSlot emitted map[CertificateKey]struct{} floor uint64 - liveSlot uint64 // trusted replay/observed watermark (NOT advanced by raw votes) - highestSlot uint64 // observability only: highest vote slot seen + liveSlot atomic.Uint64 // trusted replay/observed watermark (NOT advanced by raw votes) + highestSlot uint64 // observability only: highest vote slot seen totalPending int equivocation []EquivocationEvidence snap CertPoolSnapshot @@ -236,21 +237,24 @@ func (p *CertPool) SetEpochLookup(fn func(slot uint64) uint64) { // (from replay progress / observed finality — never from raw votes, so an // attacker cannot slide the window forward). Monotonic. func (p *CertPool) NoteLiveSlot(slot uint64) { - p.mu.Lock() - if slot > p.liveSlot { - p.liveSlot = slot + // Replay must not wait for BLS verification under p.mu merely to announce + // progress. Concurrent trusted updates may arrive out of order. + for current := p.liveSlot.Load(); slot > current; current = p.liveSlot.Load() { + if p.liveSlot.CompareAndSwap(current, slot) { + return + } } - p.mu.Unlock() } // windowAnchorLocked is the trusted upper anchor of the live vote window: the // higher of the finalized floor and the replay-observed live slot. It is NOT // derived from raw votes, so ingest cannot advance it. func (p *CertPool) windowAnchorLocked() uint64 { - if p.floor > p.liveSlot { + liveSlot := p.liveSlot.Load() + if p.floor > liveSlot { return p.floor } - return p.liveSlot + return liveSlot } // setForSlotLocked resolves the validator set covering slot. Returns nil (votes @@ -560,7 +564,7 @@ func (p *CertPool) Snapshot() CertPoolSnapshot { snap.Slots = len(p.slots) snap.Floor = p.floor snap.HighestSlot = p.highestSlot - snap.LiveSlot = p.liveSlot + snap.LiveSlot = p.liveSlot.Load() snap.PendingTotal = p.totalPending return snap } diff --git a/pkg/alpenglow/certpool_progress_test.go b/pkg/alpenglow/certpool_progress_test.go new file mode 100644 index 000000000..6732dce4b --- /dev/null +++ b/pkg/alpenglow/certpool_progress_test.go @@ -0,0 +1,112 @@ +package alpenglow + +import ( + "sync" + "testing" + "time" +) + +func TestCertPoolProgressDoesNotWaitForVerificationLock(t *testing.T) { + pool := NewCertPool(DefaultCertPoolConfig(), NewCertificateVerifier(), nil) + pool.mu.Lock() // The same lock held while verifying incoming BLS votes. + done := make(chan struct{}) + go func() { + pool.NoteLiveSlot(123) + close(done) + }() + completed := false + select { + case <-done: + completed = true + case <-time.After(time.Second): + } + pool.mu.Unlock() + <-done + if !completed { + t.Fatal("trusted replay progress waited for the verification lock") + } + if got := pool.Snapshot().LiveSlot; got != 123 { + t.Fatalf("live slot = %d, want 123", got) + } +} + +func TestCertPoolConcurrentProgressRemainsMonotonic(t *testing.T) { + pool := NewCertPool(DefaultCertPoolConfig(), NewCertificateVerifier(), nil) + const workers, updates = 16, 256 + start := make(chan struct{}) + var wg sync.WaitGroup + for worker := range workers { + wg.Go(func() { + <-start + for n := range updates { + slot := uint64(n*workers + worker + 1) + pool.NoteLiveSlot(slot) + pool.NoteLiveSlot(slot / 2) // Stale updates race newer progress. + } + }) + } + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + close(start) + var previous uint64 + for { + live := pool.Snapshot().LiveSlot + pool.mu.Lock() + anchor := pool.windowAnchorLocked() + pool.mu.Unlock() + if live < previous || anchor < live { + t.Errorf("progress regressed: previous=%d snapshot=%d anchor=%d", previous, live, anchor) + } + previous = anchor + select { + case <-finished: + pool.NoteLiveSlot(0) + pool.NoteLiveSlot(1) + if got := pool.Snapshot().LiveSlot; got != workers*updates { + t.Fatalf("live slot = %d, want %d", got, workers*updates) + } + return + default: + } + } +} + +func TestCertPoolProgressPreservesTrustedVoteWindow(t *testing.T) { + set, keys := testBLSValidatorSet(100, 40, 30, 15, 10, 5) + verifier := NewCertificateVerifier() + if err := verifier.SetValidatorSet(set); err != nil { + t.Fatal(err) + } + pool := NewCertPool(CertPoolConfig{MaxSlotsAhead: 10}, verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return set.Epoch }) + pool.NoteLiveSlot(100) + checkVote := func(slot uint64, rejected bool, live uint64) { + t.Helper() + before := pool.Snapshot() + addVote(t, pool, NewSkipVote(slot), 4, keys[4]) + after := pool.Snapshot() + wantRejected := before.VotesRejected + if rejected { + wantRejected++ + } + if after.VotesRejected != wantRejected { + t.Fatalf("slot %d: rejected=%d, want %d", slot, after.VotesRejected, wantRejected) + } + if after.LiveSlot != live { + t.Fatalf("raw vote at %d moved trusted progress to %d, want %d", slot, after.LiveSlot, live) + } + } + checkVote(110, false, 100) // Inclusive upper edge. + checkVote(111, true, 100) // Accepted raw votes cannot slide the window. + pool.NoteLiveSlot(90) + checkVote(111, true, 100) + pool.NoteLiveSlot(101) + checkVote(111, false, 101) + pool.ObserveFloor(120) + checkVote(120, true, 101) // Finalized floor still rejects old votes. + checkVote(130, false, 101) // Floor can anchor the window above replay. + checkVote(131, true, 101) +} From ad2cd47a8b2ddd36adedab75aa98e9182d1e52bf Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:40:11 -0500 Subject: [PATCH 02/13] Cache observer statistics until certificate or replay inputs change --- pkg/alpenglow/observer.go | 17 +++++- pkg/alpenglow/observer_bench_test.go | 22 +++++++ pkg/alpenglow/observer_stats_test.go | 85 ++++++++++++++++++++++++++++ 3 files changed, 121 insertions(+), 3 deletions(-) create mode 100644 pkg/alpenglow/observer_bench_test.go create mode 100644 pkg/alpenglow/observer_stats_test.go diff --git a/pkg/alpenglow/observer.go b/pkg/alpenglow/observer.go index 9b84e58cb..d9c6d3f85 100644 --- a/pkg/alpenglow/observer.go +++ b/pkg/alpenglow/observer.go @@ -95,6 +95,11 @@ type Observer struct { replayBlocks map[uint64]BlockID replayOrder []uint64 replayChecks map[CertificateKey]certificateReplayCheck + // Votes do not change certificate/replay reconciliation. Reuse its exact + // statistics until one of those inputs changes instead of scanning every + // retained certificate for each incoming vote. + pendingStats certificateReplayPendingStats + pendingStatsValid bool votesObserved uint64 certificatesObserved uint64 @@ -211,6 +216,7 @@ func (o *Observer) ObserveCertificate(cert Certificate) (Observation, error) { key := cert.Key() _, exists := o.certificates[key] if !exists { + o.pendingStatsValid = false tracked := o.trackCertificateLocked(key, cert) o.certificatesObserved++ o.applyCertificateLocked(cert) @@ -225,6 +231,7 @@ func (o *Observer) ObserveCertificate(cert Certificate) (Observation, error) { func (o *Observer) ObserveReplayBlock(obs ReplayBlockObservation) Observation { o.mu.Lock() defer o.mu.Unlock() + o.pendingStatsValid = false if obs.At.IsZero() { obs.At = time.Now() @@ -260,8 +267,8 @@ func (o *Observer) ObserveReplayResult(obs ReplayResultObservation) Observation } func (o *Observer) Snapshot() Snapshot { - o.mu.RLock() - defer o.mu.RUnlock() + o.mu.Lock() + defer o.mu.Unlock() return o.snapshotLocked() } @@ -454,7 +461,11 @@ func (o *Observer) certificateReplayPendingStatsLocked() certificateReplayPendin } func (o *Observer) snapshotLocked() Snapshot { - pending := o.certificateReplayPendingStatsLocked() + if !o.pendingStatsValid { + o.pendingStats = o.certificateReplayPendingStatsLocked() + o.pendingStatsValid = true + } + pending := o.pendingStats return Snapshot{ VotesObserved: o.votesObserved, CertificatesObserved: o.certificatesObserved, diff --git a/pkg/alpenglow/observer_bench_test.go b/pkg/alpenglow/observer_bench_test.go new file mode 100644 index 000000000..8ff14d395 --- /dev/null +++ b/pkg/alpenglow/observer_bench_test.go @@ -0,0 +1,22 @@ +package alpenglow + +import "testing" + +func BenchmarkObserverVoteWithRetainedCertificates(b *testing.B) { + o := NewObserver() + for slot := uint64(1); slot <= DefaultMaxTrackedCertificates; slot++ { + _, err := o.ObserveCertificate(Certificate{Type: CertificateNotarize, Slot: slot, + BlockHash: testHash(1), IncludedStake: 80, TotalStake: 100}) + if err != nil { + b.Fatal(err) + } + } + msg := VoteMessage{Vote: NewSkipVote(5000), Rank: 1} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := o.ObserveVote(msg); err != nil { + b.Fatal(err) + } + } +} diff --git a/pkg/alpenglow/observer_stats_test.go b/pkg/alpenglow/observer_stats_test.go new file mode 100644 index 000000000..3000de923 --- /dev/null +++ b/pkg/alpenglow/observer_stats_test.go @@ -0,0 +1,85 @@ +package alpenglow + +import ( + "fmt" + "math/rand" + "sync" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestObserverPendingStatsMatchFullScan(t *testing.T) { + // Exercise eviction, duplicate certificates, out-of-order and hashless + // replay, and retained matches/mismatches with and without retention. + for _, retention := range []int{0, 1, 8, 64} { + t.Run(fmt.Sprint(retention), func(t *testing.T) { + o := NewObserverWithConfig(ObserverConfig{MaxTrackedVotes: 8, + MaxTrackedCertificates: retention, MaxTrackedReplayBlocks: retention}) + rng := rand.New(rand.NewSource(37)) + for i := 0; i < 1000; i++ { + slot := uint64(rng.Intn(80) + 1) + switch rng.Intn(5) { + case 0, 1: + typ := CertificateNotarize + if i%3 == 0 { + typ = CertificateFinalizeFast + } + _, err := o.ObserveCertificate(Certificate{Type: typ, Slot: slot, + BlockHash: testHash(byte(slot%3 + 1)), IncludedStake: 80, TotalStake: 100}) + require.NoError(t, err) + case 2: + block := BlockID{Slot: slot} + if i%5 != 0 { + block.Hash = testHash(byte(slot%4 + 1)) + } + o.ObserveReplayBlock(ReplayBlockObservation{Block: block}) + case 3: + _, err := o.ObserveVote(VoteMessage{Vote: NewSkipVote(slot), Rank: 1}) + require.NoError(t, err) + case 4: + o.ObserveReplayResult(ReplayResultObservation{Slot: slot}) + } + o.Snapshot() + o.mu.RLock() + cached, scanned := o.pendingStats, o.certificateReplayPendingStatsLocked() + o.mu.RUnlock() + require.Equal(t, scanned, cached, "operation %d", i) + } + }) + } +} + +func TestObserverConcurrentSnapshotsAndReconciliation(t *testing.T) { + o := NewObserver() + var wg sync.WaitGroup + for worker := 0; worker < 4; worker++ { + wg.Add(1) + go func(worker int) { + defer wg.Done() + for slot := uint64(1); slot <= 100; slot++ { + switch worker { + case 0: + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: slot, Hash: testHash(1)}}) + case 1: + _, err := o.ObserveCertificate(Certificate{Type: CertificateNotarize, Slot: slot, + BlockHash: testHash(1), IncludedStake: 80, TotalStake: 100}) + if err != nil { + t.Error(err) + } + case 2: + _, err := o.ObserveVote(VoteMessage{Vote: NewSkipVote(slot), Rank: 1}) + if err != nil { + t.Error(err) + } + case 3: + o.Snapshot() + } + } + }(worker) + } + wg.Wait() + snapshot := o.Snapshot() + require.Equal(t, uint64(100), snapshot.CertificateReplayMatches) + require.Zero(t, snapshot.CertificateReplayPending) +} From 343a661af33ed8f3dcfcbcf85d2995a2794772f0 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:40:11 -0500 Subject: [PATCH 03/13] Verify certificates outside the pool mutex and reuse verified BLS points --- docs/certpool-offlock.md | 36 ++ docs/certpool-point-reuse.md | 39 ++ pkg/alpenglow/certpool.go | 426 ++++++++++++++------- pkg/alpenglow/certpool_bench_test.go | 123 ++++++ pkg/alpenglow/certpool_concurrency_test.go | 184 +++++++++ pkg/alpenglow/certpool_points_test.go | 102 +++++ 6 files changed, 762 insertions(+), 148 deletions(-) create mode 100644 docs/certpool-offlock.md create mode 100644 docs/certpool-point-reuse.md create mode 100644 pkg/alpenglow/certpool_bench_test.go create mode 100644 pkg/alpenglow/certpool_concurrency_test.go create mode 100644 pkg/alpenglow/certpool_points_test.go diff --git a/docs/certpool-offlock.md b/docs/certpool-offlock.md new file mode 100644 index 000000000..c1a9ce5d6 --- /dev/null +++ b/docs/certpool-offlock.md @@ -0,0 +1,36 @@ +# Incoming vote verification outside the pool lock + +The incoming vote pool previously held one mutex while checking BLS batches and folding signatures into aggregates. A 45-second trace on the live Zen 5 validator observed a 9.49 ms p95 acquisition wait and a 57.68 ms maximum across 34,580 normally completed AddVote calls. This blocked other incoming votes and readers; durable pruning could also wait for this lock while holding the consensus output mutex. + +The change retains one expensive BLS batch at a time using a separate verification mutex, while releasing the pool-state mutex around cryptography. One caller owns processing for each slot. Ordinary arrivals may join that slot's pending maps while verification runs, and the owner revisits state after those arrivals. Admission that needs authentication to resolve quota pressure or competing signatures waits instead of weakening bounds or first-packet-poisoning checks. + +Candidates remain pending until verification finishes, so in-flight votes count against admission limits and exact duplicates consume no additional space. Only selected candidates are removed on completion. Pruning and eviction may remove a slot during verification; pointer identity checks prevent stale work from resurrecting it or releasing accounting twice. If the epoch lookup, installed validator-set identity or shred version changes, the old slot state is retired instead of mixing bindings. Normal engine validator sets are immutable within an epoch. + +Each completed verified batch publishes promptly, outside both mutexes. New arrivals do not defer an authenticated quorum until the slot stops receiving votes. Reward-footer flushing waits for the active owner, drains relevant pending votes, and honors the publication barrier. Verified stake reads preserve freshness by waiting for that slot's owner. Snapshot counters and durable pruning do not wait for cryptography. + +Point reuse is included: aggregation uses already-verified signature points, failed batches subdivide parsed members, and the unused tally public-key aggregate is removed. Randomized coefficients, signature checks, stake thresholds, equivocation budgets and paired-vote disjointness remain. + +## Measurements + +Native AMD Ryzen 7 9700X, Go 1.26.4, with the validator and its normal continuous load running. Diagnostic processes used Nice 15; compilation used GOMAXPROCS 2. Diagnostic intervals are excluded from live comparisons. + +Point reuse alone, four alternating samples per version, median full pending-vote fold: + +| Batch size | Original | Point reuse | Time reduction | +| --- | ---: | ---: | ---: | +| 1 | 0.681 ms | 0.655 ms | 3.8% | +| 8 | 2.567 ms | 2.357 ms | 8.2% | +| 32 | 8.460 ms | 7.580 ms | 10.4% | +| 64 | 16.637 ms | 15.127 ms | 9.1% | + +The concurrent diagnostic uses eight producers submitting 64 votes, samples Snapshot latency, then flushes and checks that all 64 votes published exactly once. In two alternating samples per version, Snapshot p95 was 8.57–8.72 ms with point reuse alone and 0.008–0.038 ms with off-lock verification. Benchmark ns/op includes deliberately spaced sampling; it is not production throughput. + +The combined build deployed September 14 at 22:46 UTC. The 45-second live trace measured 36,133 normally completed incoming calls: initial mutex wait median 1.24us, p95 3.64us, p99 9.10us and maximum 0.369ms. None waited over 1ms, versus 11,742 in the earlier baseline. Different live windows and probe overhead are limitations; this is not an equivalent end-to-end voting speedup. Verification-gate and admission-condition waits are separate from the measured initial mutex acquisition. Two durable-floor calls waited at most 1.95us. + +Live voting/FAST results, exact deployment evidence and raw measurements are in the workspace at `mithril-run-20260911/certpool-offlock-20260914/RESULTS.md`. Native artifacts are under `/srv/mithril-certpool-offlock-20260914`. Four TPU workers, two shred workers, GOMAXPROCS 8, reserved vote history, previous live integrations, continuous load policy and signing/payer ledgers are preserved. + +## Validation + +New deterministic race tests park real work at the verification gate and cover concurrent admission, in-flight accounting, duplicate admission, capacity wakeup, pruning, eviction, changed bindings and reward-flush waiting. Existing invalid-share cancellation, malformed signature, aggregate-certificate, equivocation, fallback and publication tests remain. + +Native race tests passed for Alpenglow, consensus, replay and node integration. Peer isolation tests separately passed three native runs; an intermittent timeout had reproduced on the unchanged local baseline earlier. Native vet and the application build passed. The four deployed source/test files match local hashes. Instruction probes and nearby stack/register operations were checked against old/new disassembly, normalizing linker relocations. A live sanity capture observed 133 replays and 133 notarize events with no reservation exhaustion or admission rejection. diff --git a/docs/certpool-point-reuse.md b/docs/certpool-point-reuse.md new file mode 100644 index 000000000..cb99f6008 --- /dev/null +++ b/docs/certpool-point-reuse.md @@ -0,0 +1,39 @@ +# Reuse verified BLS signature points in the incoming vote pool + +This records the isolated first step. It was subsequently measured on Zen 5 and included in the [off-lock verification deployment](certpool-offlock.md); the local measurements and validation history below are retained. + +Incoming vote verification parsed each BLS signature, returned the original wire message, and parsed the same signature again when folding it into the tally. Failed aggregate checks recursively reparsed their subsets too. The tally also accumulated a public-key sum that was never read. + +The pool now returns the verified parsed members and uses their signature points directly for tally aggregation. Failed batches subdivide parsed members, and the individual-verification fallback returns verified parsed members as well. The unused tally public-key aggregate is removed; the randomized verifier's essential weighted public-key aggregate remains. + +This is the first, contained optimization from the September 14 certificate-pool investigation. The shared mutex, thresholds, publication ordering, pending limits, stake accounting, duplicate/equivocation checks, paired-vote disjointness, subgroup/infinity checks, and randomized coefficients remain in place. Moving verification outside the lock is a separate future change. Installed validator sets already cache parsed public keys. + +## Benchmark + +Local Apple M4 Pro, darwin/arm64, Go 1.26.4. `BenchmarkCertPoolFoldVerifiedBatch` exercises pending-map setup, verification, tally aggregation and accounting for valid same-payload batches, with installed public-key caches. Signing and fixture preparation are outside the timer. Each benchmark runs one goroutine with `-test.cpu=1` and `-test.benchtime=500ms`. + +The baseline is the pre-change production source at commit `0af2e094`, with the identical benchmark added. Separate before/after test binaries were built before measurement. Four measurements per version alternate in before/after/after/before order twice; the table shows medians. Tests and compilation finished before benchmarking. + +| Votes per batch | Before | After | Time reduction | Allocations before → after | +| --- | ---: | ---: | ---: | ---: | +| 1 | 0.985 ms | 0.939 ms | 4.7% | 80 → 73 | +| 8 | 3.845 ms | 3.606 ms | 6.2% | 222 → 208 | +| 32 | 12.724 ms | 11.089 ms | 12.8% | 679 → 641 | +| 64 | 25.290 ms | 21.449 ms | 15.2% | 1,263 → 1,193 | + +These are local component measurements, not Zen 5 or live FAST results. No validator restart/deployment, load-policy change, or live profiling was performed for this implementation. Large-batch time improves by several milliseconds locally, but this does not establish the reduction in lock waits or end-to-end voting latency. + +Raw samples, binary hashes and the runner are preserved in the workspace under `mithril-run-20260911/non-replay-latency-20260914/point-reuse/` and `benchmark-points.py` in its parent directory. + +## Validation + +Passed: + +* All targeted `TestCertPool` and `TestIndividuallyVerifiedBatch` tests, including three runs with race detection. +* New differential coverage checks retained points/messages against individual verification for valid batches, wrong-payload signatures in both recursive halves, malformed encodings, infinity and invalid ranks. The individual-verification fallback is checked separately. +* Existing cancellation-attack, invalid-candidate poisoning, aggregate-certificate verification, pruning, equivocation and reward-publication barrier coverage. +* Full `pkg/consensus` race suite. +* `pkg/alpenglow` race suite excluding `TestVotorBroadcasterIsolatesBlockedPeer`. +* `go vet ./pkg/alpenglow ./pkg/consensus`, `go build ./cmd/mithril`, and `git diff --check`. + +The unfiltered Alpenglow race suite hit an intermittent timeout in `TestVotorBroadcasterIsolatesBlockedPeer/reconnect`, at `peer_sender_test.go:209` waiting for the stalled connection to close. The same timeout reproduced with the unchanged certificate-pool source through a Go overlay. No transport code or timeout was changed to mask it. This pre-existing failure remains an explicit qualification limitation; the full suite is not reported as passing. diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index 50a5c5957..393880b5b 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -110,14 +110,12 @@ type tally struct { pending map[uint16]map[[sha256.Size]byte]VoteMessage // unverified candidates, by rank and signature verified map[uint16]struct{} // ranks folded into the aggregate aggSig bls12381.G2Affine // sum of verified signatures - aggPub bls12381.G1Affine // sum of verified pubkeys stake uint64 // verified stake } func newTally() *tally { t := &tally{pending: make(map[uint16]map[[sha256.Size]byte]VoteMessage), verified: make(map[uint16]struct{})} t.aggSig.SetInfinity() - t.aggPub.SetInfinity() return t } @@ -137,7 +135,11 @@ type tallyKey struct { } type poolSlot struct { - tallies map[tallyKey]*tally + // One caller owns folding for a slot while other callers may buffer votes. + // Both fields, like the maps below, are protected by CertPool.mu. + processing bool + dirty bool + tallies map[tallyKey]*tally // verifiedHash tracks the block hashes a rank has cast VERIFIED votes for, // per (rank, type), for equivocation/vote-budget enforcement. Populated only // after signature verification — never from raw ingest — so a bogus vote can @@ -167,16 +169,20 @@ type CertPool struct { // end and must not make downstream consensus decisions itself. verifiedVoteSink func(VerifiedVote) - mu sync.Mutex - epochForSlot func(slot uint64) uint64 - slots map[uint64]*poolSlot - emitted map[CertificateKey]struct{} - floor uint64 - liveSlot atomic.Uint64 // trusted replay/observed watermark (NOT advanced by raw votes) - highestSlot uint64 // observability only: highest vote slot seen - totalPending int - equivocation []EquivocationEvidence - snap CertPoolSnapshot + mu sync.Mutex + workCond *sync.Cond + verificationMu sync.Mutex // Keep expensive batch concurrency bounded at one. + batchesVerified atomic.Uint64 + epochGeneration uint64 + epochForSlot func(slot uint64) uint64 + slots map[uint64]*poolSlot + emitted map[CertificateKey]struct{} + floor uint64 + liveSlot atomic.Uint64 // trusted replay/observed watermark (NOT advanced by raw votes) + highestSlot uint64 // observability only: highest vote slot seen + totalPending int + equivocation []EquivocationEvidence + snap CertPoolSnapshot publicationMu sync.Mutex publicationCond *sync.Cond @@ -215,6 +221,7 @@ func NewCertPool(cfg CertPoolConfig, verifier *CertificateVerifier, emit func(Ce publicationCompleted: make(map[uint64]struct{}), } p.publicationCond = sync.NewCond(&p.publicationMu) + p.workCond = sync.NewCond(&p.mu) return p } @@ -230,6 +237,7 @@ func (p *CertPool) SetVerifiedVoteSink(sink func(VerifiedVote)) { func (p *CertPool) SetEpochLookup(fn func(slot uint64) uint64) { p.mu.Lock() p.epochForSlot = fn + p.epochGeneration++ p.mu.Unlock() } @@ -283,37 +291,50 @@ func (p *CertPool) AddVote(msg VoteMessage) { } slot := msg.Vote.Slot - var emits []Certificate - var verified []VerifiedVote p.mu.Lock() - if slot <= p.floor { - p.snap.VotesRejected++ - p.mu.Unlock() - return - } - // Window anchored to the TRUSTED watermark (floor / replay-observed), not to - // the highest vote slot seen — otherwise an attacker could slide it forward - // vote by vote and retain arbitrarily many future slots. - if anchor := p.windowAnchorLocked(); anchor > 0 && slot > anchor+p.cfg.MaxSlotsAhead { - p.snap.VotesRejected++ - p.mu.Unlock() - return - } - - ps := p.slots[slot] - if ps == nil { - // Hard global bound on retained slots (independent of the window). - if len(p.slots) >= p.cfg.MaxLiveSlots && !p.evictFartherFutureSlotLocked(slot) { + var ps *poolSlot + for { + if slot <= p.floor { p.snap.VotesRejected++ p.mu.Unlock() return } - ps = &poolSlot{ - tallies: make(map[tallyKey]*tally), - verifiedHash: make(map[voteDedupKey][]solana.Hash), - pendingByRank: make(map[uint16]int), + // Window anchored to the TRUSTED watermark (floor / replay-observed), not to + // the highest vote slot seen — otherwise an attacker could slide it forward + // vote by vote and retain arbitrarily many future slots. + if anchor := p.windowAnchorLocked(); anchor > 0 && slot > anchor+p.cfg.MaxSlotsAhead { + p.snap.VotesRejected++ + p.mu.Unlock() + return } - p.slots[slot] = ps + + ps = p.slots[slot] + if ps == nil { + // Hard global bound on retained slots (independent of the window). + if len(p.slots) >= p.cfg.MaxLiveSlots && !p.evictFartherFutureSlotLocked(slot) { + p.snap.VotesRejected++ + p.mu.Unlock() + return + } + ps = &poolSlot{ + tallies: make(map[tallyKey]*tally), + verifiedHash: make(map[voteDedupKey][]solana.Hash), + pendingByRank: make(map[uint16]int), + } + p.slots[slot] = ps + } + // Ordinary arrivals can join the bounded pending maps during BLS work. + // Quota pressure and competing signatures still wait for authentication + // before admission, preserving the first-packet-poisoning protections. + if ps.processing && p.admissionNeedsFoldLocked(ps, msg) { + p.workCond.Wait() + continue + } + break + } + owner := !ps.processing + if owner { + ps.processing = true } tk := tallyKey{Type: msg.Vote.Type, Hash: msg.Vote.BlockHash} @@ -336,7 +357,11 @@ func (p *CertPool) AddVote(msg VoteMessage) { // the rank quota, authenticate that rank's parked candidates and free // the invalid ones before deciding whether the real vote has room. if set := p.setForSlotLocked(slot); set != nil { - verified = append(verified, p.foldPendingRankLocked(slot, ps, msg.Rank, set)...) + p.foldPendingRankLocked(slot, ps, msg.Rank, set) + } + if p.slots[slot] != ps { + p.finishSlotLocked(slot, ps, nil) + return } candidates = tl.pending[msg.Rank] } @@ -346,7 +371,11 @@ func (p *CertPool) AddVote(msg VoteMessage) { atCap := ps.pendingCount >= p.cfg.MaxPendingVotesPerSlot || p.totalPending >= p.cfg.MaxPendingVotesTotal if !rejectIncoming && atCap { if set := p.setForSlotLocked(slot); set != nil { - verified = append(verified, p.foldAllPendingLocked(slot, ps, set)...) + p.foldAllPendingLocked(slot, ps, set) + } + if p.slots[slot] != ps { + p.finishSlotLocked(slot, ps, nil) + return } if p.totalPending >= p.cfg.MaxPendingVotesTotal { p.evictFartherFutureSlotLocked(slot) @@ -386,54 +415,116 @@ func (p *CertPool) AddVote(msg VoteMessage) { if slot > p.highestSlot { p.highestSlot = slot } + if !owner { + ps.dirty = true + p.mu.Unlock() + return + } if forceFold { if set := p.setForSlotLocked(slot); set != nil { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.foldTallyLocked(slot, ps, tl, set) + } + } + emits := p.drainSlotLocked(slot, ps, false) + p.finishSlotLocked(slot, ps, emits) +} + +func (p *CertPool) admissionNeedsFoldLocked(ps *poolSlot, msg VoteMessage) bool { + tl := ps.tallies[tallyKey{Type: msg.Vote.Type, Hash: msg.Vote.BlockHash}] + if tl != nil { + if _, done := tl.verified[msg.Rank]; done { + return false + } + candidates := tl.pending[msg.Rank] + if _, duplicate := candidates[sha256.Sum256(msg.Signature)]; duplicate { + return false + } + if len(candidates) > 0 { + return true } } - // Keep verified stake fresh for the Votor fallback triggers. Only plain - // notarize/skip arrivals can change the trigger predicates. - if msg.Vote.Type == VoteTypeNotarize || msg.Vote.Type == VoteTypeSkip { - verified = append(verified, p.maybeFoldTriggersLocked(slot, ps)...) + return ps.pendingByRank[msg.Rank] >= p.cfg.MaxPendingVotesPerRankSlot || + ps.pendingCount >= p.cfg.MaxPendingVotesPerSlot || p.totalPending >= p.cfg.MaxPendingVotesTotal +} + +// waitForSlotLocked releases mu while the current owner finishes. Pruning can +// remove or replace the slot, so callers always use the returned current state. +func (p *CertPool) waitForSlotLocked(slot uint64) *poolSlot { + for { + ps := p.slots[slot] + if ps == nil || !ps.processing { + return ps + } + p.workCond.Wait() } - var assembledVerified []VerifiedVote - emits, assembledVerified = p.maybeAssembleLocked(slot, ps) - verified = append(verified, assembledVerified...) - sink := p.verifiedVoteSink - publication := p.reservePublicationLocked(verified) - p.mu.Unlock() +} - p.publishVerifiedVotes(sink, verified, publication) +// drainSlotLocked catches arrivals buffered while the owner was outside mu. +// Sub-threshold votes remain lazy except when preparing a reward footer. +func (p *CertPool) drainSlotLocked(slot uint64, ps *poolSlot, flush bool) []Certificate { + var emits []Certificate + for p.slots[slot] == ps { + ps.dirty = false + if flush { + if set := p.setForSlotLocked(slot); set != nil { + for key, tl := range ps.tallies { + if key.Type == VoteTypeSkip || key.Type == VoteTypeNotarize { + p.foldTallyLocked(slot, ps, tl, set) + } + } + } + } + p.maybeFoldTriggersLocked(slot, ps) + certs := p.maybeAssembleLocked(slot, ps) + emits = append(emits, certs...) + if !ps.dirty { + break + } + } + return emits +} + +// finishSlotLocked takes the publication barrier before making the slot +// available to a flushing caller, then releases mu and emits certificates. +func (p *CertPool) finishSlotLocked(slot uint64, ps *poolSlot, emits []Certificate) uint64 { + if p.slots[slot] != ps { + emits = nil + } + target := p.publicationTargetLocked() + ps.processing = false + p.workCond.Broadcast() + p.mu.Unlock() for _, cert := range emits { p.emitCert(cert) } + return target } // OnValidatorSetInstalled retries assembly for buffered slots that resolve to // the newly-installed epoch. Requires a real slot→epoch lookup; without one no // slot can be safely attributed to the epoch, so nothing is retried. func (p *CertPool) OnValidatorSetInstalled(epoch uint64) { - var emits []Certificate - var verified []VerifiedVote p.mu.Lock() + var slots []uint64 if p.epochForSlot != nil { - for slot, ps := range p.slots { - if p.epochForSlot(slot) != epoch { - continue + for slot := range p.slots { + if p.epochForSlot(slot) == epoch { + slots = append(slots, slot) } - verified = append(verified, p.maybeFoldTriggersLocked(slot, ps)...) - certs, newlyVerified := p.maybeAssembleLocked(slot, ps) - emits = append(emits, certs...) - verified = append(verified, newlyVerified...) } } - sink := p.verifiedVoteSink - publication := p.reservePublicationLocked(verified) p.mu.Unlock() - p.publishVerifiedVotes(sink, verified, publication) - for _, cert := range emits { - p.emitCert(cert) + for _, slot := range slots { + p.mu.Lock() + ps := p.waitForSlotLocked(slot) + if ps == nil || p.epochForSlot == nil || p.epochForSlot(slot) != epoch { + p.mu.Unlock() + continue + } + ps.processing = true + emits := p.drainSlotLocked(slot, ps, false) + p.finishSlotLocked(slot, ps, emits) } } @@ -442,34 +533,19 @@ func (p *CertPool) OnValidatorSetInstalled(epoch uint64) { // calls this just before building the slot+8 footer so valid below-threshold // votes are not omitted from reward certificates. func (p *CertPool) FlushRewardVotes(slot uint64) { - var emits []Certificate - var verified []VerifiedVote p.mu.Lock() - ps := p.slots[slot] - set := p.setForSlotLocked(slot) - if ps != nil && set != nil { - for key, tl := range ps.tallies { - if key.Type == VoteTypeSkip || key.Type == VoteTypeNotarize { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) - } - } - var assembled []VerifiedVote - emits, assembled = p.maybeAssembleLocked(slot, ps) - verified = append(verified, assembled...) - } - sink := p.verifiedVoteSink - publication := p.reservePublicationLocked(verified) - targetPublication := p.publicationTargetLocked() - p.mu.Unlock() - - p.publishVerifiedVotes(sink, verified, publication) - for _, cert := range emits { - p.emitCert(cert) + ps := p.waitForSlotLocked(slot) + if ps == nil { + target := p.publicationTargetLocked() + p.mu.Unlock() + p.waitForPublication(target) + return } - // A vote can finish BLS verification in AddVote just before this flush takes - // the pool lock, then still be in flight to the reward builder. Wait through - // that publication sequence so a footer cannot omit an already-verified vote. - p.waitForPublication(targetPublication) + ps.processing = true + emits := p.drainSlotLocked(slot, ps, true) + target := p.finishSlotLocked(slot, ps, emits) + // Includes publication by an owner that was verifying when flush arrived. + p.waitForPublication(target) } // reservePublicationLocked assigns ordering while p.mu is held. Flush can then @@ -545,6 +621,7 @@ func (p *CertPool) ObserveFloor(finalizedSlot uint64) { delete(p.emitted, key) } } + p.workCond.Broadcast() } p.mu.Unlock() } @@ -561,6 +638,7 @@ func (p *CertPool) Snapshot() CertPoolSnapshot { p.mu.Lock() defer p.mu.Unlock() snap := p.snap + snap.BatchesVerified = p.batchesVerified.Load() snap.Slots = len(p.slots) snap.Floor = p.floor snap.HighestSlot = p.highestSlot @@ -652,10 +730,13 @@ func meets(f Fraction, stake, total uint64) bool { // implementation (Agave) would. This is the ONLY fold policy: one fork-choice // behavior for observer and voting nodes alike; sub-trigger tallies still // cost nothing. -func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) []VerifiedVote { +func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) { + if p.slots[slot] != ps { + return + } set := p.setForSlotLocked(slot) if set == nil { - return nil // votes stay buffered; retried on OnValidatorSetInstalled + return // votes stay buffered; retried on OnValidatorSetInstalled } total := set.TotalStake v := buildTriggerViewLocked(ps, set) @@ -690,22 +771,21 @@ func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) []Verified } if !foldSkip && !foldAllNotar && len(foldNotar) == 0 { - return nil + return } - var verified []VerifiedVote for tk, tl := range ps.tallies { switch tk.Type { case VoteTypeNotarize: if foldAllNotar || foldNotar[tk.Hash] { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.foldTallyLocked(slot, ps, tl, set) } case VoteTypeSkip: if foldSkip { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.foldTallyLocked(slot, ps, tl, set) } } } - return verified + return } // VotorStakes is the verified-stake observation Votor's fallback-trigger @@ -727,12 +807,12 @@ type VotorStakes struct { func (p *CertPool) VerifiedVotorStakes(slot uint64) (VotorStakes, bool) { p.mu.Lock() defer p.mu.Unlock() + ps := p.waitForSlotLocked(slot) set := p.setForSlotLocked(slot) if set == nil { return VotorStakes{}, false } out := VotorStakes{Notarize: make(map[solana.Hash]uint64), TotalStake: set.TotalStake} - ps := p.slots[slot] if ps == nil { return out, true } @@ -808,14 +888,16 @@ func targetsForSlot(ps *poolSlot) []certTarget { // maybeAssembleLocked checks every assemblable target for the slot: folds // pending votes (batch verification) once candidate stake crosses the // threshold, and returns any newly assembled certificates for emission. -func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate, []VerifiedVote) { +func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) []Certificate { + if p.slots[slot] != ps { + return nil + } set := p.setForSlotLocked(slot) if set == nil { - return nil, nil // validator set / epoch not resolvable yet; votes stay buffered + return nil // validator set / epoch not resolvable yet; votes stay buffered } var emits []Certificate - var verified []VerifiedVote for _, target := range targetsForSlot(ps) { key := CertificateKey{Type: target.certType, Slot: slot} if target.certType.HasBlock() { @@ -850,8 +932,11 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate } // Candidate stake crossed: fold pending votes (one pairing per tally). - verified = append(verified, p.foldTallyLocked(slot, ps, base, set)...) - verified = append(verified, p.foldTallyLocked(slot, ps, fb, set)...) + p.foldTallyLocked(slot, ps, base, set) + p.foldTallyLocked(slot, ps, fb, set) + if p.slots[slot] != ps { + return nil + } verifiedStake := uint64(0) if base != nil { @@ -873,7 +958,7 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate p.snap.CertsEmitted++ emits = append(emits, cert) } - return emits, verified + return emits } // foldTallyLocked batch-verifies a tally's pending votes. All sign the same @@ -882,9 +967,9 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) ([]Certificate // vote verifies does it update the durable per-slot state — the vote-budget / // equivocation ledger (verifiedHash) and base↔fallback disjointness — so raw // votes can never poison those. -func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *ValidatorSet) []VerifiedVote { - if tl == nil || len(tl.pending) == 0 { - return nil +func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *ValidatorSet) { + if p.slots[slot] != ps || tl == nil || len(tl.pending) == 0 { + return } batch := make([]VoteMessage, 0, pendingCandidateCount(tl)) for rank, candidates := range tl.pending { @@ -893,6 +978,7 @@ func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *Va for _, msg := range candidates { batch = append(batch, msg) } + continue } else { p.snap.VotesRejected += uint64(count) } @@ -904,10 +990,50 @@ func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *Va } p.totalPending -= count } + // Keep candidates in the pending maps while verifying. Concurrent arrivals + // can deduplicate against them, and every in-flight byte still consumes the + // normal per-rank, per-slot and global admission budget. + generation := p.epochGeneration + shredVersion := p.verifier.ShredVersion() + p.mu.Unlock() + p.verificationMu.Lock() good := p.verifyBatch(batch, set) + p.verificationMu.Unlock() + p.mu.Lock() + if p.slots[slot] != ps { + return // pruning/eviction already released the pending accounting + } + current := p.setForSlotLocked(slot) + if generation != p.epochGeneration || shredVersion != p.verifier.ShredVersion() || !sameInstalledValidatorSet(current, set) { + // Do not mix an old aggregate with a new epoch/key/stake binding. Retire + // this slot's old state; a later arrival starts afresh with the current set. + p.totalPending -= ps.pendingCount + delete(p.slots, slot) + for key := range p.emitted { + if key.Slot == slot { + delete(p.emitted, key) + } + } + p.workCond.Broadcast() + return + } + for _, msg := range batch { + candidates := tl.pending[msg.Rank] + delete(candidates, sha256.Sum256(msg.Signature)) + if len(candidates) == 0 { + delete(tl.pending, msg.Rank) + } + ps.pendingCount-- + ps.pendingByRank[msg.Rank]-- + if ps.pendingByRank[msg.Rank] == 0 { + delete(ps.pendingByRank, msg.Rank) + } + p.totalPending-- + } verified := make([]VerifiedVote, 0, len(good)) - for _, msg := range good { + for i := range good { + msg := good[i].message verified = append(verified, VerifiedVote{ Message: msg, Result: VoteVerifyResult{ @@ -955,40 +1081,43 @@ func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *Va } } - pub, err := validatorBLSPubkey(*set, int(msg.Rank)) - if err != nil { - continue - } - var sig bls12381.G2Affine - if _, err := sig.SetBytes(msg.Signature); err != nil { - continue - } - tl.aggPub.Add(&tl.aggPub, &pub) - tl.aggSig.Add(&tl.aggSig, &sig) + // Reuse the exact signature point authenticated by verifyBatch. + tl.aggSig.Add(&tl.aggSig, &good[i].sig) tl.verified[msg.Rank] = struct{}{} tl.stake += set.Validators[msg.Rank].Stake ps.verifiedHash[dk] = append(seen, msg.Vote.BlockHash) } p.snap.BadSignatures += uint64(len(batch) - len(good)) - return verified + // Publish this batch before verifying any newly buffered work. In particular, + // a growing slot must not hold back an already-authenticated quorum. + sink := p.verifiedVoteSink + publication := p.reservePublicationLocked(verified) + p.mu.Unlock() + p.publishVerifiedVotes(sink, verified, publication) + p.mu.Lock() } -func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, set *ValidatorSet) []VerifiedVote { - var verified []VerifiedVote +// Installed sets own immutable parsed-key arrays. Reinstallation, even for the +// same epoch and keys, gets a new array and therefore invalidates in-flight work. +func sameInstalledValidatorSet(a, b *ValidatorSet) bool { + return a != nil && b != nil && a.Epoch == b.Epoch && len(a.parsedPubkeys) > 0 && + len(a.parsedPubkeys) == len(b.parsedPubkeys) && &a.parsedPubkeys[0] == &b.parsedPubkeys[0] +} + +func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, set *ValidatorSet) { for _, tl := range ps.tallies { if len(tl.pending[rank]) != 0 { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.foldTallyLocked(slot, ps, tl, set) } } - return verified + return } -func (p *CertPool) foldAllPendingLocked(slot uint64, ps *poolSlot, set *ValidatorSet) []VerifiedVote { - var verified []VerifiedVote +func (p *CertPool) foldAllPendingLocked(slot uint64, ps *poolSlot, set *ValidatorSet) { for _, tl := range ps.tallies { - verified = append(verified, p.foldTallyLocked(slot, ps, tl, set)...) + p.foldTallyLocked(slot, ps, tl, set) } - return verified + return } func (p *CertPool) evictFartherFutureSlotLocked(incoming uint64) bool { @@ -1021,6 +1150,7 @@ func (p *CertPool) evictFartherFutureSlotLocked(incoming uint64) bool { p.totalPending = 0 } delete(p.slots, victim) + p.workCond.Broadcast() return true } @@ -1049,13 +1179,13 @@ type parsedBatchVote struct { // check, bisecting on failure. Unweighted aggregation is insufficient here: // invalid shares can cancel while the downstream pool later keeps only a subset. // Independent random coefficients bind success to every individual member. -func (p *CertPool) verifyBatch(batch []VoteMessage, set *ValidatorSet) []VoteMessage { +func (p *CertPool) verifyBatch(batch []VoteMessage, set *ValidatorSet) []parsedBatchVote { if len(batch) == 0 { return nil } - p.snap.BatchesVerified++ payload, err := EncodeVotePayloadToSign(batch[0].Vote, p.verifier.ShredVersion()) if err != nil { + p.batchesVerified.Add(1) return nil } @@ -1072,27 +1202,35 @@ func (p *CertPool) verifyBatch(batch []VoteMessage, set *ValidatorSet) []VoteMes members = append(members, parsedBatchVote{message: msg, pubkey: pub, sig: sig}) } + return p.verifyParsedBatch(members, payload) +} + +// verifyParsedBatch keeps the owned parsed points through failed-batch +// subdivision and returns only individually bound, verified members. Each +// subdivision still uses fresh random coefficients; parsing is the only work +// reused. Verification does not read or mutate the pool's slot maps. +func (p *CertPool) verifyParsedBatch(members []parsedBatchVote, payload []byte) []parsedBatchVote { + p.batchesVerified.Add(1) if len(members) == 0 { return nil } if len(members) == 1 { if aggregatePairingOK(members[0].pubkey, payload, members[0].sig) { - return []VoteMessage{members[0].message} + return members } return nil } if ok, err := randomizedAggregatePairingOK(members, payload); err == nil && ok { - return batchMessages(members) + return members } else if err != nil { // Entropy failure must reduce performance, never verification strength. return individuallyVerifiedBatch(members, payload) } // Aggregate failed: bisect the structurally-valid subset. - messages := batchMessages(members) - mid := len(messages) / 2 - valid := p.verifyBatch(messages[:mid], set) - valid = append(valid, p.verifyBatch(messages[mid:], set)...) + mid := len(members) / 2 + valid := p.verifyParsedBatch(members[:mid:mid], payload) + valid = append(valid, p.verifyParsedBatch(members[mid:], payload)...) return valid } @@ -1128,24 +1266,16 @@ func randomNonzeroBatchCoefficient() (*big.Int, error) { } } -func individuallyVerifiedBatch(members []parsedBatchVote, payload []byte) []VoteMessage { - valid := make([]VoteMessage, 0, len(members)) +func individuallyVerifiedBatch(members []parsedBatchVote, payload []byte) []parsedBatchVote { + valid := make([]parsedBatchVote, 0, len(members)) for i := range members { if aggregatePairingOK(members[i].pubkey, payload, members[i].sig) { - valid = append(valid, members[i].message) + valid = append(valid, members[i]) } } return valid } -func batchMessages(members []parsedBatchVote) []VoteMessage { - messages := make([]VoteMessage, len(members)) - for i := range members { - messages[i] = members[i].message - } - return messages -} - func aggregatePairingOK(aggPub bls12381.G1Affine, payload []byte, aggSig bls12381.G2Affine) bool { if aggPub.IsInfinity() { return false diff --git a/pkg/alpenglow/certpool_bench_test.go b/pkg/alpenglow/certpool_bench_test.go new file mode 100644 index 000000000..a52fb3718 --- /dev/null +++ b/pkg/alpenglow/certpool_bench_test.go @@ -0,0 +1,123 @@ +package alpenglow + +import ( + "crypto/sha256" + "fmt" + "slices" + "sync" + "sync/atomic" + "testing" + "time" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" + "github.com/gagliardetto/solana-go" +) + +// Benchmark the full pending-vote fold, including verification, aggregation and +// accounting. Keys and signatures are prepared outside the timed region. +func BenchmarkCertPoolFoldVerifiedBatch(b *testing.B) { + for _, size := range []int{1, 8, 32, 64} { + b.Run(fmt.Sprintf("votes=%d", size), func(b *testing.B) { + verifier, installed, vote, batch := certPoolBenchmarkFixture(b, size) + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return installed.Epoch }) + tl := newTally() + ps := &poolSlot{verifiedHash: make(map[voteDedupKey][]solana.Hash), pendingByRank: make(map[uint16]int)} + pool.slots[vote.Slot] = ps + for _, msg := range batch { + tl.pending[msg.Rank] = map[[sha256.Size]byte]VoteMessage{sha256.Sum256(msg.Signature): msg} + ps.pendingByRank[msg.Rank]++ + ps.pendingCount++ + pool.totalPending++ + } + pool.mu.Lock() + pool.foldTallyLocked(vote.Slot, ps, tl, &installed) + pool.mu.Unlock() + if len(tl.verified) != size || tl.stake != uint64(size) || pool.totalPending != 0 { + b.Fatal("incomplete verified fold") + } + } + }) + } +} + +func certPoolBenchmarkFixture(b *testing.B, size int) (*CertificateVerifier, ValidatorSet, Vote, []VoteMessage) { + stakes := make([]uint64, size) + for i := range stakes { + stakes[i] = 1 + } + set, keys := testBLSValidatorSet(uint64(size), stakes...) + verifier := NewCertificateVerifier() + if err := verifier.SetValidatorSet(set); err != nil { + b.Fatal(err) + } + // Exercise the same cached public-key representation used in production. + installed, ok := verifier.ValidatorSetForEpoch(set.Epoch) + if !ok { + b.Fatal("missing installed validator set") + } + vote := NewSkipVote(500) + payload, err := EncodeVotePayloadToSign(vote, verifier.ShredVersion()) + if err != nil { + b.Fatal(err) + } + point, err := bls12381.HashToG2(payload, []byte(blsHashToPointDST)) + if err != nil { + b.Fatal(err) + } + batch := make([]VoteMessage, size) + for i := range batch { + var sig bls12381.G2Affine + sig.ScalarMultiplication(&point, keys[i]) + raw := sig.RawBytes() + batch[i] = VoteMessage{Vote: vote, Rank: uint16(i), Signature: raw[:]} + } + return verifier, installed, vote, batch +} + +// Observe short pool reads while eight peer-like producers submit one slot. +// The latency metrics, not ns/op (which includes deliberate sampling delays), +// measure whether cryptographic work blocks unrelated pool readers. +func BenchmarkCertPoolConcurrentVotes(b *testing.B) { + verifier, installed, vote, batch := certPoolBenchmarkFixture(b, 64) + waits := make([]int64, 0, 16*b.N) + b.ResetTimer() + for n := 0; n < b.N; n++ { + pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return installed.Epoch }) + pool.NoteLiveSlot(vote.Slot) + var verified atomic.Int64 + pool.SetVerifiedVoteSink(func(VerifiedVote) { verified.Add(1) }) + start := make(chan struct{}) + var producers sync.WaitGroup + for worker := 0; worker < 8; worker++ { + producers.Add(1) + go func(worker int) { + defer producers.Done() + <-start + for i := worker; i < len(batch); i += 8 { + pool.AddVote(batch[i]) + } + }(worker) + } + close(start) + for sample := 0; sample < 16; sample++ { + time.Sleep(100 * time.Microsecond) + t := time.Now() + pool.Snapshot() + waits = append(waits, time.Since(t).Nanoseconds()) + } + producers.Wait() + pool.FlushRewardVotes(vote.Slot) + if verified.Load() != 64 || pool.Snapshot().PendingTotal != 0 { + b.Fatal("lost or duplicated votes") + } + } + b.StopTimer() + slices.Sort(waits) + b.ReportMetric(float64(waits[len(waits)*95/100]), "snapshot-p95-ns") + b.ReportMetric(float64(waits[len(waits)-1]), "snapshot-max-ns") +} diff --git a/pkg/alpenglow/certpool_concurrency_test.go b/pkg/alpenglow/certpool_concurrency_test.go new file mode 100644 index 000000000..b1d4bc253 --- /dev/null +++ b/pkg/alpenglow/certpool_concurrency_test.go @@ -0,0 +1,184 @@ +package alpenglow + +import ( + "sync" + "testing" + "time" +) + +func waitCertPoolCall(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("certificate-pool operation did not finish") + } +} + +func certPoolAsync(fn func()) <-chan struct{} { + done := make(chan struct{}) + go func() { defer close(done); fn() }() + return done +} + +// Park a real owner at the production verification gate. No cryptography is +// stubbed; releasing the gate runs the normal randomized verification path. +func parkCertPoolVerification(t *testing.T, pool *CertPool, msg VoteMessage) (func(), <-chan struct{}) { + t.Helper() + pool.verificationMu.Lock() + var once sync.Once + release := func() { once.Do(pool.verificationMu.Unlock) } + t.Cleanup(release) + done := certPoolAsync(func() { pool.AddVote(msg) }) + ready := certPoolAsync(func() { + for pool.Snapshot().PendingTotal == 0 { + time.Sleep(time.Millisecond) + } + }) + waitCertPoolCall(t, ready) // Also proves Snapshot can acquire mu during work. + return release, done +} + +func TestCertPoolOffLockAdmissionAndRewardFlush(t *testing.T) { + pool, set, keys, emitted := newTestPool(t) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + vote := NewSkipVote(500) + first := VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])} + second := VoteMessage{Vote: vote, Rank: 1, Signature: signTestVote(t, vote, keys[1])} + release, owner := parkCertPoolVerification(t, pool, first) + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(second) })) + if got := pool.Snapshot().PendingTotal; got != 2 { + t.Fatalf("pending = %d; in-flight and newly buffered votes must both count", got) + } + flush := certPoolAsync(func() { pool.FlushRewardVotes(vote.Slot) }) + select { + case <-flush: + t.Fatal("flush escaped in-flight verification") + case <-time.After(20 * time.Millisecond): + } + if len(seen) != 0 { + t.Fatal("unverified vote reached sink") + } + release() + waitCertPoolCall(t, owner) + waitCertPoolCall(t, flush) + if len(seen) != 2 || pool.Snapshot().PendingTotal != 0 { + t.Fatalf("flush lost or duplicated work: published=%d snapshot=%+v", len(seen), pool.Snapshot()) + } + if len(*emitted) == 0 { + t.Fatal("buffered arrivals did not drive certificate assembly") + } + for _, cert := range *emitted { + if _, _, err := verifyCertificateWithSet(set, cert, true); err != nil { + t.Fatalf("concurrently assembled certificate failed verification: %v", err) + } + } +} + +func TestCertPoolOffLockPruningDiscardsInFlightResults(t *testing.T) { + pool, _, keys, emitted := newTestPool(t) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + vote := NewSkipVote(500) + release, owner := parkCertPoolVerification(t, pool, VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])}) + waitCertPoolCall(t, certPoolAsync(func() { pool.ObserveFloor(vote.Slot) })) + if got := pool.Snapshot(); got.PendingTotal != 0 || got.Slots != 0 { + t.Fatalf("pruning retained in-flight accounting: %+v", got) + } + release() + waitCertPoolCall(t, owner) + if len(seen) != 0 || len(*emitted) != 0 || pool.Snapshot().PendingTotal != 0 { + t.Fatal("pruned work was published or decremented accounting twice") + } + addVote(t, pool, NewSkipVote(501), 0, keys[0]) + if len(seen) != 1 { + t.Fatal("subsequent live vote was lost") + } +} + +func TestCertPoolOffLockBindingChangeDiscardsResults(t *testing.T) { + for _, kind := range []string{"validator-set", "epoch-lookup"} { + t.Run(kind, func(t *testing.T) { + pool, set, keys, emitted := newTestPool(t) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + vote := NewSkipVote(500) + msg := VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])} + release, owner := parkCertPoolVerification(t, pool, msg) + if kind == "validator-set" { + if err := pool.verifier.SetValidatorSet(set); err != nil { + t.Fatal(err) + } + } else { + pool.SetEpochLookup(func(uint64) uint64 { return set.Epoch }) + } + release() + waitCertPoolCall(t, owner) + if got := pool.Snapshot(); len(seen) != 0 || len(*emitted) != 0 || got.PendingTotal != 0 || got.Slots != 0 { + t.Fatalf("stale binding published or retained state: %+v", got) + } + pool.AddVote(msg) + if len(seen) != 1 { + t.Fatal("vote could not be retried under the current binding") + } + }) + } +} + +func TestCertPoolOffLockPendingBoundAndDuplicates(t *testing.T) { + pool, _, keys, _ := newTestPool(t) + pool.cfg.MaxPendingVotesPerSlot = 2 + pool.cfg.MaxPendingVotesTotal = 2 + vote := NewSkipVote(500) + msgs := make([]VoteMessage, 3) + for i := range msgs { + msgs[i] = VoteMessage{Vote: vote, Rank: uint16(i), Signature: signTestVote(t, vote, keys[i])} + } + release, owner := parkCertPoolVerification(t, pool, msgs[0]) + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(msgs[1]) })) + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(msgs[0]) })) + third := certPoolAsync(func() { pool.AddVote(msgs[2]) }) + select { + case <-third: + t.Fatal("capacity-pressure admission did not wait for authentication") + case <-time.After(20 * time.Millisecond): + } + if got := pool.Snapshot().PendingTotal; got != 2 { + t.Fatalf("in-flight votes escaped bounds or duplicate consumed capacity: %d", got) + } + release() + waitCertPoolCall(t, owner) + waitCertPoolCall(t, third) + pool.FlushRewardVotes(vote.Slot) + if got := pool.Snapshot(); got.PendingTotal != 0 || got.VotesAccepted != 3 { + t.Fatalf("capacity wakeup lost or duplicated votes: %+v", got) + } +} + +func TestCertPoolOffLockEvictionDoesNotResurrectSlot(t *testing.T) { + set, keys := testBLSValidatorSet(100, 40, 30, 15, 10, 5) + verifier := NewCertificateVerifier() + if err := verifier.SetValidatorSet(set); err != nil { + t.Fatal(err) + } + pool := NewCertPool(CertPoolConfig{MaxLiveSlots: 1}, verifier, nil) + pool.SetEpochLookup(func(uint64) uint64 { return set.Epoch }) + pool.NoteLiveSlot(100) + seen := make(chan VerifiedVote, 8) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { seen <- v }) + future := NewSkipVote(110) + release, owner := parkCertPoolVerification(t, pool, VoteMessage{Vote: future, Rank: 0, Signature: signTestVote(t, future, keys[0])}) + near := NewSkipVote(101) + msg := VoteMessage{Vote: near, Rank: 4, Signature: signTestVote(t, near, keys[4])} + waitCertPoolCall(t, certPoolAsync(func() { pool.AddVote(msg) })) + if got := pool.Snapshot(); got.PendingTotal != 1 || got.Slots != 1 { + t.Fatalf("eviction accounting mismatch: %+v", got) + } + release() + waitCertPoolCall(t, owner) + pool.FlushRewardVotes(near.Slot) + if len(seen) != 1 || (<-seen).Message.Vote.Slot != near.Slot || pool.Snapshot().PendingTotal != 0 { + t.Fatal("evicted work displaced or corrupted the nearer slot") + } +} diff --git a/pkg/alpenglow/certpool_points_test.go b/pkg/alpenglow/certpool_points_test.go new file mode 100644 index 000000000..4f76ad01e --- /dev/null +++ b/pkg/alpenglow/certpool_points_test.go @@ -0,0 +1,102 @@ +package alpenglow + +import ( + "bytes" + "testing" + + bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" +) + +func TestCertPoolVerifiedPointsMatchIndividualVerification(t *testing.T) { + for _, mixed := range []bool{false, true} { + name := "valid" + if mixed { + name = "mixed-invalid" + } + t.Run(name, func(t *testing.T) { + pool, set, keys, _ := newTestPool(t) + vote := NewSkipVote(500) + var batch []VoteMessage + for i, key := range keys { + signedVote := vote + if mixed && i%2 == 1 { + signedVote = NewSkipVote(501) // Valid point, wrong payload, in both halves. + } + batch = append(batch, VoteMessage{Vote: vote, Rank: uint16(i), Signature: signTestVote(t, signedVote, key)}) + } + if mixed { + var infinity bls12381.G2Affine + infinity.SetInfinity() + raw := infinity.RawBytes() + batch = append(batch, + VoteMessage{Vote: vote, Rank: 0, Signature: []byte{0xff}}, + VoteMessage{Vote: vote, Rank: 1, Signature: raw[:]}, + VoteMessage{Vote: vote, Rank: uint16(len(keys)), Signature: batch[0].Signature}, + ) + } + var expected []VoteMessage + for _, msg := range batch { + if _, err := verifyVoteMessageWithSet(set, msg); err == nil { + expected = append(expected, msg) + } + } + verified := pool.verifyBatch(batch, &set) + if len(verified) != len(expected) { + t.Fatalf("verified %d members, want %d", len(verified), len(expected)) + } + for i, member := range verified { + if member.message.Rank != expected[i].Rank || member.message.Vote != expected[i].Vote || !bytes.Equal(member.message.Signature, expected[i].Signature) { + t.Fatalf("member %d does not match its individually verified message", i) + } + raw := member.sig.RawBytes() + if !bytes.Equal(raw[:], expected[i].Signature) { + t.Fatalf("member %d retained the wrong signature point", i) + } + pub, err := validatorBLSPubkey(set, int(expected[i].Rank)) + if err != nil || !member.pubkey.Equal(&pub) { + t.Fatalf("member %d retained the wrong public key", i) + } + } + if mixed && pool.Snapshot().BatchesVerified <= 1 { + t.Fatal("mixed batch did not exercise recursive verification") + } + }) + } +} + +// Entropy failure uses this individual-verification path instead of accepting +// an unweighted aggregate. Reusing points must preserve its filtering too. +func TestIndividuallyVerifiedBatchRetainsOnlyValidPoints(t *testing.T) { + _, set, keys, _ := newTestPool(t) + vote := NewSkipVote(502) + payload, err := EncodeVotePayloadToSign(vote, 0) + if err != nil { + t.Fatal(err) + } + var members []parsedBatchVote + for i, key := range keys { + signedVote := vote + if i%2 == 1 { + signedVote = NewSkipVote(503) + } + msg := VoteMessage{Vote: vote, Rank: uint16(i), Signature: signTestVote(t, signedVote, key)} + pub, err := validatorBLSPubkey(set, i) + if err != nil { + t.Fatal(err) + } + var sig bls12381.G2Affine + if _, err := sig.SetBytes(msg.Signature); err != nil { + t.Fatal(err) + } + members = append(members, parsedBatchVote{message: msg, pubkey: pub, sig: sig}) + } + verified := individuallyVerifiedBatch(members, payload) + if len(verified) != 3 { + t.Fatalf("verified %d, want 3", len(verified)) + } + for i, member := range verified { + if member.message.Rank != uint16(i*2) || !member.sig.Equal(&members[i*2].sig) || !member.pubkey.Equal(&members[i*2].pubkey) { + t.Fatalf("wrong verified member at %d", i) + } + } +} From 5133813713ac85eeebbd1decd45e05bdcb492b6c Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:45:28 -0500 Subject: [PATCH 04/13] Document isolated PR validation and retain relevant benchmark evidence --- .gitattributes | 7 + .../2026-09-14/0-before.txt | 9 + ...mpare-BenchmarkCertPoolConcurrentVotes.txt | 6 + ...are-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 + ...idate-BenchmarkCertPoolConcurrentVotes.txt | 6 + ...ate-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 + .../2026-09-14/1-stage1.txt | 9 + ...idate-BenchmarkCertPoolConcurrentVotes.txt | 6 + ...ate-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 + .../2026-09-14/2-stage1.txt | 9 + .../2026-09-14/3-before.txt | 9 + ...mpare-BenchmarkCertPoolConcurrentVotes.txt | 6 + ...are-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 + .../2026-09-14/4-before.txt | 9 + .../2026-09-14/5-stage1.txt | 9 + .../2026-09-14/6-stage1.txt | 9 + .../2026-09-14/7-before.txt | 9 + .../2026-09-14/README.md | 1 + .../2026-09-14/stage1-benchmarks.json | 246 ++++++++++++++++++ .../pr-split-2026-09-15/certificate/README.md | 7 + .../certificate/certificate-tests.log | 1 + .../certificate/certificate-vet.log | 0 22 files changed, 394 insertions(+) create mode 100644 .gitattributes create mode 100644 docs/results/certificate-processing/2026-09-14/0-before.txt create mode 100644 docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt create mode 100644 docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt create mode 100644 docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt create mode 100644 docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt create mode 100644 docs/results/certificate-processing/2026-09-14/1-stage1.txt create mode 100644 docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt create mode 100644 docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt create mode 100644 docs/results/certificate-processing/2026-09-14/2-stage1.txt create mode 100644 docs/results/certificate-processing/2026-09-14/3-before.txt create mode 100644 docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt create mode 100644 docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt create mode 100644 docs/results/certificate-processing/2026-09-14/4-before.txt create mode 100644 docs/results/certificate-processing/2026-09-14/5-stage1.txt create mode 100644 docs/results/certificate-processing/2026-09-14/6-stage1.txt create mode 100644 docs/results/certificate-processing/2026-09-14/7-before.txt create mode 100644 docs/results/certificate-processing/2026-09-14/README.md create mode 100644 docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json create mode 100644 docs/results/pr-split-2026-09-15/certificate/README.md create mode 100644 docs/results/pr-split-2026-09-15/certificate/certificate-tests.log create mode 100644 docs/results/pr-split-2026-09-15/certificate/certificate-vet.log diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..d64360d9a --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Keep raw benchmark evidence available without overwhelming the review. +# Retain raw Go CPU padding and test-framework space/tab indentation. +docs/results/**/*.json linguist-generated=true +docs/results/**/*.jsonl linguist-generated=true +docs/results/**/*.txt linguist-generated=true whitespace=-blank-at-eol +docs/results/**/*.log linguist-generated=true -whitespace +docs/results/**/*.tar.gz linguist-generated=true diff --git a/docs/results/certificate-processing/2026-09-14/0-before.txt b/docs/results/certificate-processing/2026-09-14/0-before.txt new file mode 100644 index 000000000..886c64f84 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/0-before.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 894 663584 ns/op 10417 B/op 80 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 232 2555742 ns/op 33852 B/op 222 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 70 8424882 ns/op 120269 B/op 679 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 34 16426787 ns/op 232923 B/op 1263 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt new file mode 100644 index 000000000..a9488ca64 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt @@ -0,0 +1,6 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolConcurrentVotes-8 18 31244638 ns/op 9756702 snapshot-max-ns 8565800 snapshot-p95-ns 276815 B/op 1762 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt new file mode 100644 index 000000000..d828970a0 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 951 641175 ns/op 10537 B/op 80 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 259 2300836 ns/op 33003 B/op 215 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 80 7344967 ns/op 116161 B/op 648 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14097224 ns/op 224464 B/op 1200 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt new file mode 100644 index 000000000..b90df1c00 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt @@ -0,0 +1,6 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolConcurrentVotes-8 34 17816437 ns/op 199885 snapshot-max-ns 38282 snapshot-p95-ns 255776 B/op 1579 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt new file mode 100644 index 000000000..d0bb23c76 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 943 636360 ns/op 10857 B/op 83 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 262 2291882 ns/op 33323 B/op 218 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 80 7409936 ns/op 116481 B/op 651 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14327493 ns/op 224784 B/op 1203 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/1-stage1.txt b/docs/results/certificate-processing/2026-09-14/1-stage1.txt new file mode 100644 index 000000000..c6f4b9ae4 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/1-stage1.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 915 660591 ns/op 9689 B/op 73 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 252 2363508 ns/op 32155 B/op 208 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 79 7789873 ns/op 115313 B/op 641 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 39 15996987 ns/op 223619 B/op 1193 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt new file mode 100644 index 000000000..a46f8dcc7 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt @@ -0,0 +1,6 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolConcurrentVotes-8 34 16300179 ns/op 148197 snapshot-max-ns 8365 snapshot-p95-ns 255690 B/op 1575 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt new file mode 100644 index 000000000..960594bc4 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 924 702853 ns/op 10857 B/op 83 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 246 2295439 ns/op 33323 B/op 218 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 81 7358090 ns/op 116481 B/op 651 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14125492 ns/op 224784 B/op 1203 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/2-stage1.txt b/docs/results/certificate-processing/2026-09-14/2-stage1.txt new file mode 100644 index 000000000..f3f370099 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/2-stage1.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 872 652789 ns/op 9689 B/op 73 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 258 2472982 ns/op 32155 B/op 208 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 67 7768642 ns/op 115309 B/op 641 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 43 15097852 ns/op 223616 B/op 1193 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/3-before.txt b/docs/results/certificate-processing/2026-09-14/3-before.txt new file mode 100644 index 000000000..c39fbbfe2 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/3-before.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 856 662411 ns/op 10417 B/op 80 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 232 2661079 ns/op 33851 B/op 222 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 63 9372424 ns/op 120270 B/op 679 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 32 17757518 ns/op 232925 B/op 1263 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt new file mode 100644 index 000000000..09a8cf4ae --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt @@ -0,0 +1,6 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolConcurrentVotes-8 18 31545982 ns/op 9839357 snapshot-max-ns 8719558 snapshot-p95-ns 276423 B/op 1761 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt new file mode 100644 index 000000000..0e7dac2bf --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 943 639930 ns/op 10537 B/op 80 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 259 2285913 ns/op 33003 B/op 215 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 79 7427797 ns/op 116161 B/op 648 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 40 14114964 ns/op 224466 B/op 1200 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/4-before.txt b/docs/results/certificate-processing/2026-09-14/4-before.txt new file mode 100644 index 000000000..78f039350 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/4-before.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 846 731522 ns/op 10417 B/op 80 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 225 2575042 ns/op 33852 B/op 222 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 69 8494169 ns/op 120269 B/op 679 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 34 16556301 ns/op 232923 B/op 1263 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/5-stage1.txt b/docs/results/certificate-processing/2026-09-14/5-stage1.txt new file mode 100644 index 000000000..7d3bb27b2 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/5-stage1.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 934 657515 ns/op 9688 B/op 73 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 258 2349599 ns/op 32155 B/op 208 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 79 7376201 ns/op 115313 B/op 641 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14172786 ns/op 223616 B/op 1193 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/6-stage1.txt b/docs/results/certificate-processing/2026-09-14/6-stage1.txt new file mode 100644 index 000000000..d2216cf06 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/6-stage1.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 873 645598 ns/op 9689 B/op 73 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 260 2332481 ns/op 32155 B/op 208 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 82 7392025 ns/op 115312 B/op 641 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 40 15156833 ns/op 223618 B/op 1193 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/7-before.txt b/docs/results/certificate-processing/2026-09-14/7-before.txt new file mode 100644 index 000000000..45dec925d --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/7-before.txt @@ -0,0 +1,9 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkCertPoolFoldVerifiedBatch/votes=1 812 697862 ns/op 10417 B/op 80 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=8 235 2558921 ns/op 33851 B/op 222 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=32 72 8400940 ns/op 120268 B/op 679 allocs/op +BenchmarkCertPoolFoldVerifiedBatch/votes=64 34 16717873 ns/op 232923 B/op 1263 allocs/op +PASS diff --git a/docs/results/certificate-processing/2026-09-14/README.md b/docs/results/certificate-processing/2026-09-14/README.md new file mode 100644 index 000000000..96003cfbb --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/README.md @@ -0,0 +1 @@ +Historical native Zen5 raw benchmarks supporting docs/certpool-offlock.md. before versus stage1 compares point reuse; stage1-compare versus candidate compares verification outside the lock. Concurrent ns/op includes deliberate sampling delays and is not throughput. Production pool source is preserved by this split; see the fresh split-branch validation separately. diff --git a/docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json b/docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json new file mode 100644 index 000000000..3d14f1300 --- /dev/null +++ b/docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json @@ -0,0 +1,246 @@ +{ + "window": { + "start_utc": "2026-09-14T22:32:50.064174+00:00", + "end_utc": "2026-09-14T22:33:24.153378+00:00" + }, + "samples": { + "1": { + "before": [ + { + "ns": 663584.0, + "bytes": 10417.0, + "allocs": 80.0 + }, + { + "ns": 662411.0, + "bytes": 10417.0, + "allocs": 80.0 + }, + { + "ns": 731522.0, + "bytes": 10417.0, + "allocs": 80.0 + }, + { + "ns": 697862.0, + "bytes": 10417.0, + "allocs": 80.0 + } + ], + "stage1": [ + { + "ns": 660591.0, + "bytes": 9689.0, + "allocs": 73.0 + }, + { + "ns": 652789.0, + "bytes": 9689.0, + "allocs": 73.0 + }, + { + "ns": 657515.0, + "bytes": 9688.0, + "allocs": 73.0 + }, + { + "ns": 645598.0, + "bytes": 9689.0, + "allocs": 73.0 + } + ] + }, + "8": { + "before": [ + { + "ns": 2555742.0, + "bytes": 33852.0, + "allocs": 222.0 + }, + { + "ns": 2661079.0, + "bytes": 33851.0, + "allocs": 222.0 + }, + { + "ns": 2575042.0, + "bytes": 33852.0, + "allocs": 222.0 + }, + { + "ns": 2558921.0, + "bytes": 33851.0, + "allocs": 222.0 + } + ], + "stage1": [ + { + "ns": 2363508.0, + "bytes": 32155.0, + "allocs": 208.0 + }, + { + "ns": 2472982.0, + "bytes": 32155.0, + "allocs": 208.0 + }, + { + "ns": 2349599.0, + "bytes": 32155.0, + "allocs": 208.0 + }, + { + "ns": 2332481.0, + "bytes": 32155.0, + "allocs": 208.0 + } + ] + }, + "32": { + "before": [ + { + "ns": 8424882.0, + "bytes": 120269.0, + "allocs": 679.0 + }, + { + "ns": 9372424.0, + "bytes": 120270.0, + "allocs": 679.0 + }, + { + "ns": 8494169.0, + "bytes": 120269.0, + "allocs": 679.0 + }, + { + "ns": 8400940.0, + "bytes": 120268.0, + "allocs": 679.0 + } + ], + "stage1": [ + { + "ns": 7789873.0, + "bytes": 115313.0, + "allocs": 641.0 + }, + { + "ns": 7768642.0, + "bytes": 115309.0, + "allocs": 641.0 + }, + { + "ns": 7376201.0, + "bytes": 115313.0, + "allocs": 641.0 + }, + { + "ns": 7392025.0, + "bytes": 115312.0, + "allocs": 641.0 + } + ] + }, + "64": { + "before": [ + { + "ns": 16426787.0, + "bytes": 232923.0, + "allocs": 1263.0 + }, + { + "ns": 17757518.0, + "bytes": 232925.0, + "allocs": 1263.0 + }, + { + "ns": 16556301.0, + "bytes": 232923.0, + "allocs": 1263.0 + }, + { + "ns": 16717873.0, + "bytes": 232923.0, + "allocs": 1263.0 + } + ], + "stage1": [ + { + "ns": 15996987.0, + "bytes": 223619.0, + "allocs": 1193.0 + }, + { + "ns": 15097852.0, + "bytes": 223616.0, + "allocs": 1193.0 + }, + { + "ns": 14172786.0, + "bytes": 223616.0, + "allocs": 1193.0 + }, + { + "ns": 15156833.0, + "bytes": 223618.0, + "allocs": 1193.0 + } + ] + } + }, + "summary": { + "1": { + "before": { + "ns": 680723.0, + "bytes": 10417.0, + "allocs": 80.0 + }, + "stage1": { + "ns": 655152.0, + "bytes": 9689.0, + "allocs": 73.0 + }, + "time_reduction_percent": 3.7564471892384987 + }, + "8": { + "before": { + "ns": 2566981.5, + "bytes": 33851.5, + "allocs": 222.0 + }, + "stage1": { + "ns": 2356553.5, + "bytes": 32155.0, + "allocs": 208.0 + }, + "time_reduction_percent": 8.197487983454499 + }, + "32": { + "before": { + "ns": 8459525.5, + "bytes": 120269.0, + "allocs": 679.0 + }, + "stage1": { + "ns": 7580333.5, + "bytes": 115312.5, + "allocs": 641.0 + }, + "time_reduction_percent": 10.392923338312531 + }, + "64": { + "before": { + "ns": 16637087.0, + "bytes": 232923.0, + "allocs": 1263.0 + }, + "stage1": { + "ns": 15127342.5, + "bytes": 223617.0, + "allocs": 1193.0 + }, + "time_reduction_percent": 9.074572369550026 + } + } +} \ No newline at end of file diff --git a/docs/results/pr-split-2026-09-15/certificate/README.md b/docs/results/pr-split-2026-09-15/certificate/README.md new file mode 100644 index 000000000..2d5c32a92 --- /dev/null +++ b/docs/results/pr-split-2026-09-15/certificate/README.md @@ -0,0 +1,7 @@ +# Review branch validation, September 15 + +This PR is split from #279 plus the later working-tree improvements. The tested code commit before this documentation commit was `d7b19cac27e9b9fb54b249b58cf2883b9ad9ecc6`. Tests ran locally on Apple M4 Pro, Go1.26.4, GOMAXPROCS3 and package parallelism2. Logs beside this file are the fresh split-branch checks, not native measurements. Existing Zen5 benchmark documents retain their original baselines and scope. No validator restart or deployment occurred during this reorganization. + +Four independent branches start at current alpenglow-dev33dde405. Voting is based on the certificate-processing PR; leader packing is based on the streaming-preparation PR. Runtime changes and status-cache changes are independent. Shared CLI/configuration additions need an ordinary three-file merge reconciliation when combining leader packing and voting. A separate audit checkout reconciled these additions and matched the preserved full implementation exactly across Go sources, module files, TOML configuration and CI. + +The branch-specific race suites and vet passed. The combined audit has a separately documented pre-existing intermittent peer reconnect timeout; this is not reported as an entirely green combined race run. diff --git a/docs/results/pr-split-2026-09-15/certificate/certificate-tests.log b/docs/results/pr-split-2026-09-15/certificate/certificate-tests.log new file mode 100644 index 000000000..0d2a04a55 --- /dev/null +++ b/docs/results/pr-split-2026-09-15/certificate/certificate-tests.log @@ -0,0 +1 @@ +ok github.com/Overclock-Validator/mithril/pkg/alpenglow 8.507s diff --git a/docs/results/pr-split-2026-09-15/certificate/certificate-vet.log b/docs/results/pr-split-2026-09-15/certificate/certificate-vet.log new file mode 100644 index 000000000..e69de29bb From 45fcf669be26eadbdda41b9983c302d7bd505da4 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:42:47 -0500 Subject: [PATCH 05/13] Use bounded multi-scalar aggregation for larger vote batches Keep full-field independent coefficients and existing verification admission. Use sequential G1/G2 MultiExp with one arithmetic task for at least 16 members, retaining the scalar path where setup costs dominate. Preserve individual fallback and failed-batch subdivision. Validate invalid members, wrong payloads and cancelling shares across both paths. Document local component and execution-contention measurements; native and live validation remain pending. --- docs/certpool-offlock.md | 52 ++++++++++++++++++++++++++++ pkg/alpenglow/certpool.go | 40 +++++++++++++++++++++ pkg/alpenglow/certpool_bench_test.go | 39 ++++++++++++++++++++- pkg/alpenglow/certpool_test.go | 48 +++++++++++++++++++++++++ 4 files changed, 178 insertions(+), 1 deletion(-) diff --git a/docs/certpool-offlock.md b/docs/certpool-offlock.md index c1a9ce5d6..3012f0c4e 100644 --- a/docs/certpool-offlock.md +++ b/docs/certpool-offlock.md @@ -34,3 +34,55 @@ Live voting/FAST results, exact deployment evidence and raw measurements are in New deterministic race tests park real work at the verification gate and cover concurrent admission, in-flight accounting, duplicate admission, capacity wakeup, pruning, eviction, changed bindings and reward-flush waiting. Existing invalid-share cancellation, malformed signature, aggregate-certificate, equivocation, fallback and publication tests remain. Native race tests passed for Alpenglow, consensus, replay and node integration. Peer isolation tests separately passed three native runs; an intermittent timeout had reproduced on the unchanged local baseline earlier. Native vet and the application build passed. The four deployed source/test files match local hashes. Instruction probes and nearby stack/register operations were checked against old/new disassembly, normalizing linker relocations. A live sanity capture observed 133 replays and 133 notarize events with no reservation exhaustion or admission rejection. + +## Bounded multi-scalar aggregation + +Batches of at least 16 parsed votes now use gnark `MultiExp` for each weighted +public-key/signature sum. G1 and G2 run sequentially with `NbTasks: 1`; the +existing pool verification mutex still admits one expensive batch at a time. +Smaller batches keep the scalar loop because bucket setup costs more than it +saves, particularly during failed-batch subdivision and two-candidate checks. + +Each member still receives a fresh, independent, nonzero coefficient sampled +from the full scalar field. Its public key and signature use the same +coefficient. Entropy/aggregation errors retain the individual-verification +fallback. Parsing, subgroup/infinity checks, failed-batch subdivision, +publication and stake accounting are unchanged. + +### Local component measurements + +Apple M4 Pro, Go 1.26.4, one caller, GOMAXPROCS=12. Three samples per version; +medians below compare the split branch before this change with bounded +multi-scalar aggregation. `BenchmarkCertPoolFoldVerifiedBatch` includes parsing, +pending-map setup, verification and tally folding; fixture signing is excluded. + +| Votes in fold | Scalar implementation | Bounded MultiExp | +| --- | ---: | ---: | +| 32 | 11.23 ms | 6.73 ms | +| 64 | 21.26 ms | 10.61 ms | + +A same-binary comparison of the weighted-pairing helpers found 2-vote batches +slower with MultiExp (1.49 → 2.04 ms) and 4-vote batches slower (2.01 → 2.44 ms). +Those sizes therefore retain the scalar implementation in production. At 64 +votes, allocated bytes per full fold increased from approximately 225 KB to +274 KB, although allocation count decreased from 1,204 to approximately 922. + +Run `go test ./pkg/alpenglow -run '^$' -bench '^BenchmarkCertPool(WeightedPairing|FoldVerifiedBatch)$' -benchmem -benchtime=500ms -count=3`. + +These are local component measurements; this aggregation change has not been +deployed and does not yet have native Zen 5 or live FAST results. The earlier +native/deployment measurements above describe the preceding implementation. +The full local Alpenglow race suite and vet pass. Additional coverage checks +invalid members at every position, wrong payloads, cancelling invalid shares +in larger batches, and subdivision across the scalar/MultiExp boundary. + +The 16-vote cutoff was also measured directly: weighted pairing improved from +5.28 ms to 3.80 ms. A separate local scheduling experiment alternated baseline, +candidate, candidate, baseline with 64 valid votes every 200 ms alongside +4,096 repeated transfer executions. With GOMAXPROCS=2, transfer execution was +11.89–12.04 ms before and 11.80–11.83 ms after; certificate processing was +24.75–26.15 ms before and 12.94–13.24 ms after. With GOMAXPROCS=1, transfer +execution improved from 15.75–15.91 ms to 13.95–15.13 ms, but certificate wall +time stayed around 30–32 ms. Scheduler waiting therefore still matters. This +controlled experiment excludes account commits, network delivery and live +vote persistence; it does not establish production latency or FAST scores. diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index 393880b5b..69c0e15d3 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -8,6 +8,7 @@ import ( "sync" "sync/atomic" + "github.com/Overclock-Validator/gnark-crypto/ecc" bls12381 "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381" blsfr "github.com/Overclock-Validator/gnark-crypto/ecc/bls12-381/fr" "github.com/Overclock-Validator/mithril/pkg/mlog" @@ -1235,6 +1236,45 @@ func (p *CertPool) verifyParsedBatch(members []parsedBatchVote, payload []byte) } func randomizedAggregatePairingOK(members []parsedBatchVote, payload []byte) (bool, error) { + // Bucket setup outweighs MultiExp's savings on small batches, including + // the two-candidate collision path and failed-batch subdivisions. + if len(members) < 16 { + return randomizedAggregatePairingScalarOK(members, payload) + } + return randomizedAggregatePairingMultiExpOK(members, payload) +} + +func randomizedAggregatePairingMultiExpOK(members []parsedBatchVote, payload []byte) (bool, error) { + pubkeys := make([]bls12381.G1Affine, len(members)) + signatures := make([]bls12381.G2Affine, len(members)) + coefficients := make([]blsfr.Element, len(members)) + for i := range members { + coefficient, err := randomNonzeroBatchCoefficient() + if err != nil { + return false, err + } + // Keep independent, nonzero, full-field coefficients and apply the + // same coefficient to each member's public key and signature. + coefficients[i].SetBigInt(coefficient) + pubkeys[i] = members[i].pubkey + signatures[i] = members[i].sig + } + // MultiExp defaults to using all CPUs. Keep its arithmetic concurrency at + // one, and run G1/G2 sequentially, to avoid competing with replay workers. + // Pool-level verification admission remains bounded by verificationMu. + config := ecc.MultiExpConfig{NbTasks: 1} + var aggPub bls12381.G1Affine + if _, err := aggPub.MultiExp(pubkeys, coefficients, config); err != nil { + return false, err + } + var aggSig bls12381.G2Affine + if _, err := aggSig.MultiExp(signatures, coefficients, config); err != nil { + return false, err + } + return aggregatePairingOK(aggPub, payload, aggSig), nil +} + +func randomizedAggregatePairingScalarOK(members []parsedBatchVote, payload []byte) (bool, error) { var aggPub bls12381.G1Affine var aggSig bls12381.G2Affine aggPub.SetInfinity() diff --git a/pkg/alpenglow/certpool_bench_test.go b/pkg/alpenglow/certpool_bench_test.go index a52fb3718..8b29fcb37 100644 --- a/pkg/alpenglow/certpool_bench_test.go +++ b/pkg/alpenglow/certpool_bench_test.go @@ -44,7 +44,7 @@ func BenchmarkCertPoolFoldVerifiedBatch(b *testing.B) { } } -func certPoolBenchmarkFixture(b *testing.B, size int) (*CertificateVerifier, ValidatorSet, Vote, []VoteMessage) { +func certPoolBenchmarkFixture(b testing.TB, size int) (*CertificateVerifier, ValidatorSet, Vote, []VoteMessage) { stakes := make([]uint64, size) for i := range stakes { stakes[i] = 1 @@ -121,3 +121,40 @@ func BenchmarkCertPoolConcurrentVotes(b *testing.B) { b.ReportMetric(float64(waits[len(waits)*95/100]), "snapshot-p95-ns") b.ReportMetric(float64(waits[len(waits)-1]), "snapshot-max-ns") } + +func BenchmarkCertPoolWeightedPairing(b *testing.B) { + for _, size := range []int{2, 4, 8, 16, 32, 64, 128} { + verifier, set, vote, batch := certPoolBenchmarkFixture(b, size) + payload, err := EncodeVotePayloadToSign(vote, verifier.ShredVersion()) + if err != nil { + b.Fatal(err) + } + members := make([]parsedBatchVote, size) + for i, msg := range batch { + pub, err := validatorBLSPubkey(set, int(msg.Rank)) + if err != nil { + b.Fatal(err) + } + members[i] = parsedBatchVote{message: msg, pubkey: pub} + if _, err := members[i].sig.SetBytes(msg.Signature); err != nil { + b.Fatal(err) + } + } + for _, impl := range []struct { + name string + verify func([]parsedBatchVote, []byte) (bool, error) + }{ + {"Scalar", randomizedAggregatePairingScalarOK}, + {"MultiExp", randomizedAggregatePairingMultiExpOK}, + } { + b.Run(fmt.Sprintf("votes=%d/%s", size, impl.name), func(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + if ok, err := impl.verify(members, payload); err != nil || !ok { + b.Fatalf("verification: %t %v", ok, err) + } + } + }) + } + } +} diff --git a/pkg/alpenglow/certpool_test.go b/pkg/alpenglow/certpool_test.go index 8c7c36789..24970345c 100644 --- a/pkg/alpenglow/certpool_test.go +++ b/pkg/alpenglow/certpool_test.go @@ -1,6 +1,7 @@ package alpenglow import ( + "fmt" "math/big" "testing" "time" @@ -805,3 +806,50 @@ func TestCertPoolEmitsOnce(t *testing.T) { t.Fatalf("no new certs expected, went from %d to %d", n, len(*emitted)) } } + +// Exercise both aggregation paths, including a malformed member at every +// position, a different signed payload, and failed-batch subdivision. Valid +// votes must survive independently of which member causes the batch to fail. +func TestCertPoolBatchAggregationPaths(t *testing.T) { + for _, size := range []int{2, 8, 16, 64} { + t.Run(fmt.Sprintf("votes=%d", size), func(t *testing.T) { + verifier, set, vote, batch := certPoolBenchmarkFixture(t, size) + pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) + payload, err := EncodeVotePayloadToSign(vote, verifier.ShredVersion()) + if err != nil { + t.Fatal(err) + } + members := pool.verifyBatch(batch, &set) + if len(members) != size { + t.Fatal("valid batch rejected") + } + var tweak bls12381.G2Affine + tweak.ScalarMultiplicationBase(big.NewInt(1234567)) + for i := range members { + bad := append([]parsedBatchVote(nil), members...) + bad[i].sig.Add(&bad[i].sig, &tweak) + if ok, err := randomizedAggregatePairingOK(bad, payload); err != nil || ok { + t.Fatalf("bad member %d: accepted=%t err=%v", i, ok, err) + } + } + wrongPayload := append([]byte(nil), payload...) + wrongPayload[0] ^= 1 + if ok, err := randomizedAggregatePairingOK(members, wrongPayload); err != nil || ok { + t.Fatalf("wrong payload: accepted=%t err=%v", ok, err) + } + // Preserve the unweighted sum while corrupting two shares in a + // large batch; subdivision must keep exactly the honest members. + members[0].sig.Add(&members[0].sig, &tweak) + members[len(members)-1].sig.Sub(&members[len(members)-1].sig, &tweak) + valid := pool.verifyParsedBatch(members, payload) + if len(valid) != size-2 { + t.Fatalf("verified %d, want %d", len(valid), size-2) + } + for _, member := range valid { + if member.message.Rank == 0 || int(member.message.Rank) == size-1 { + t.Fatal("invalid share survived") + } + } + }) + } +} From 623dcfd90e876c04dd84871c4d91a7c85ce774b8 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:53:27 -0500 Subject: [PATCH 06/13] Preserve scalar verification on single-thread runtimes Native contention tests showed that MultiExp task handoffs can increase certificate wall time with GOMAXPROCS=1. Preserve the original scalar path there while retaining bounded MultiExp for larger batches on multi-thread runtimes. Repeat combined native race tests, adversarial tests at one and two threads, vet and build. Record both fold gains and noisy execution samples; no live voting improvement is inferred. --- docs/certpool-offlock.md | 53 ++++++++++++++++++++++++++++++--------- pkg/alpenglow/certpool.go | 7 ++++-- 2 files changed, 46 insertions(+), 14 deletions(-) diff --git a/docs/certpool-offlock.md b/docs/certpool-offlock.md index 3012f0c4e..0b8fcaaa6 100644 --- a/docs/certpool-offlock.md +++ b/docs/certpool-offlock.md @@ -37,11 +37,15 @@ Native race tests passed for Alpenglow, consensus, replay and node integration. ## Bounded multi-scalar aggregation -Batches of at least 16 parsed votes now use gnark `MultiExp` for each weighted +With more than one Go execution thread, batches of at least 16 parsed votes +use gnark `MultiExp` for each weighted public-key/signature sum. G1 and G2 run sequentially with `NbTasks: 1`; the existing pool verification mutex still admits one expensive batch at a time. Smaller batches keep the scalar loop because bucket setup costs more than it saves, particularly during failed-batch subdivision and two-candidate checks. +Single-thread configurations also retain the scalar path: native contention +tests found that MultiExp task handoffs could increase certificate wall time +there, despite reducing arithmetic. Each member still receives a fresh, independent, nonzero coefficient sampled from the full scalar field. Its public key and signature use the same @@ -70,19 +74,44 @@ votes, allocated bytes per full fold increased from approximately 225 KB to Run `go test ./pkg/alpenglow -run '^$' -bench '^BenchmarkCertPool(WeightedPairing|FoldVerifiedBatch)$' -benchmem -benchtime=500ms -count=3`. These are local component measurements; this aggregation change has not been -deployed and does not yet have native Zen 5 or live FAST results. The earlier -native/deployment measurements above describe the preceding implementation. +deployed. Native staging measurements follow below. The earlier deployment +measurements above describe the preceding implementation. The full local Alpenglow race suite and vet pass. Additional coverage checks invalid members at every position, wrong payloads, cancelling invalid shares in larger batches, and subdivision across the scalar/MultiExp boundary. The 16-vote cutoff was also measured directly: weighted pairing improved from -5.28 ms to 3.80 ms. A separate local scheduling experiment alternated baseline, -candidate, candidate, baseline with 64 valid votes every 200 ms alongside -4,096 repeated transfer executions. With GOMAXPROCS=2, transfer execution was -11.89–12.04 ms before and 11.80–11.83 ms after; certificate processing was -24.75–26.15 ms before and 12.94–13.24 ms after. With GOMAXPROCS=1, transfer -execution improved from 15.75–15.91 ms to 13.95–15.13 ms, but certificate wall -time stayed around 30–32 ms. Scheduler waiting therefore still matters. This -controlled experiment excludes account commits, network delivery and live -vote persistence; it does not establish production latency or FAST scores. +5.28 ms to 3.80 ms locally and from 3.44 ms to 2.29 ms on Zen 5. + +### Native staging validation + +AMD Ryzen 7 9700X, Go 1.26.4, GOMAXPROCS=2, Nice 15, two-core CPU quota, with +the normal validator workload still running. The baseline restores the scalar +implementation from the preceding certificate split head (`395e4566`) with +identical fixtures. Baseline/candidate/candidate/baseline runs were followed by +a final check after adding the single-thread safeguard. Full-fold ranges: + +| Votes in fold | Scalar baseline | Final implementation | +| --- | ---: | ---: | +| 32 | 7.385–7.386 ms | 4.238–4.344 ms | +| 64 | 14.217–14.280 ms | 6.800–7.135 ms | + +A controlled scheduling experiment runs 64 valid votes every 200 ms alongside +4,096 repeated transfer executions. With two Go execution threads, the final +alternating native comparison reduced certificate processing from +16.25–16.39 ms to 8.43–10.22 ms. Execution results were noisy: 13.80–14.38 ms +baseline versus 13.84–17.56 ms candidate. The earlier native comparison and the +local comparison showed no execution regression, but the slower final sample +is retained; the shared host does not establish absence of interference. +This workload excludes account commits, network delivery and vote persistence. + +The initial prototype's single-thread certificate latency could worsen while +execution improved slightly. The final implementation therefore keeps the +original scalar arithmetic whenever GOMAXPROCS is one. No worker-count or +verification-admission changes accompany this optimization. + +Final combined native race suites, targeted adversarial tests at GOMAXPROCS=1 +and 2, vet and the validator production build passed. Native source hashes +matched the complete local combined candidate. Live FAST, durable-root lag and +voting latency still need a deployment comparison; these staging results do +not establish those gains. diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index 69c0e15d3..671af66e7 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -5,6 +5,7 @@ import ( "crypto/sha256" "fmt" "math/big" + "runtime" "sync" "sync/atomic" @@ -1237,8 +1238,10 @@ func (p *CertPool) verifyParsedBatch(members []parsedBatchVote, payload []byte) func randomizedAggregatePairingOK(members []parsedBatchVote, payload []byte) (bool, error) { // Bucket setup outweighs MultiExp's savings on small batches, including - // the two-candidate collision path and failed-batch subdivisions. - if len(members) < 16 { + // the two-candidate collision path and failed-batch subdivisions. Its + // internal task handoffs can also delay verification behind replay when + // only one Go execution thread is available, despite doing less arithmetic. + if len(members) < 16 || runtime.GOMAXPROCS(0) == 1 { return randomizedAggregatePairingScalarOK(members, payload) } return randomizedAggregatePairingMultiExpOK(members, payload) From 4927628dd0bdbc2bc67fd176b9e4b841f729d6d7 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 15 Sep 2026 00:21:54 -0500 Subject: [PATCH 07/13] alpenglow: reconcile only pending observer certificates during replay --- docs/alpenglow_branch_engine.md | 25 ++++++++++ pkg/alpenglow/observer.go | 70 +++++++++++++++------------- pkg/alpenglow/observer_bench_test.go | 37 ++++++++++++++- pkg/alpenglow/observer_stats_test.go | 44 ++++++++++++++++- 4 files changed, 141 insertions(+), 35 deletions(-) diff --git a/docs/alpenglow_branch_engine.md b/docs/alpenglow_branch_engine.md index 6ca91a1f4..33bc12d7f 100644 --- a/docs/alpenglow_branch_engine.md +++ b/docs/alpenglow_branch_engine.md @@ -154,6 +154,31 @@ mixed (heterogeneous-client) or Mithril-only cluster identically: timeouts, vote signing/transmission, durable vote-history persistence, and standstill participation. +## Replay-observer diagnostics + +The observer retains certificate history for deduplication and match/mismatch +reporting. A separate bounded index contains only retained, block-bearing +certificates that have not yet been reconciled against replay. Reconciliation +removes an entry after either a match or mismatch; eviction removes it together +with the historical certificate. Hashless/skipped replay cannot reconcile a +block-bearing certificate. Pending counts and age/window statistics retain the +same semantics, but scan unresolved entries rather than completed history. + +This index is disposable, process-local diagnostic state. It neither authorizes +votes nor substitutes for verified certificates, the chain tracker's finality +checks, durable signing bounds, vote history, or checkpoint recovery. Those +checks and persistence contracts are unchanged. + +`BenchmarkObserverEmptyReplay` measures observer work for an empty block, with +or without four preceding skipped slots, against 4,096 retained certificates. +It covers 0, 32, and 4,096 unresolved entries. On Ryzen 9700X (GOMAXPROCS=8, +three 300 ms runs), median time for the four-skips-plus-empty case with 32 +unresolved entries was 686.4 µs before the index and 1.87 µs afterward. With +all 4,096 entries unresolved it was 380.5 → 159.3 µs. These are component +benchmarks; they exclude execution, certificate cryptography, network delivery, +and end-to-end FAST inclusion. Live comparisons must account for observer +history warming after a restart and different leader/skip patterns. + ## What this proves — and does not The certificate layer proves which block *data* the cluster settled on. In diff --git a/pkg/alpenglow/observer.go b/pkg/alpenglow/observer.go index d9c6d3f85..8f3470bec 100644 --- a/pkg/alpenglow/observer.go +++ b/pkg/alpenglow/observer.go @@ -95,6 +95,12 @@ type Observer struct { replayBlocks map[uint64]BlockID replayOrder []uint64 replayChecks map[CertificateKey]certificateReplayCheck + // Only retained, block-bearing certificates that have not yet been checked + // against replay belong here. Checked history stays in certificates and + // replayChecks for diagnostics/deduplication, but need not be scanned on + // every block (including skipped slots). This is an in-memory observer + // index, not voting authorization or durable crash-recovery state. + pendingReplayCertificates map[CertificateKey]BlockID // Votes do not change certificate/replay reconciliation. Reuse its exact // statistics until one of those inputs changes instead of scanning every // retained certificate for each incoming vote. @@ -154,11 +160,12 @@ func NewObserverWithConfig(cfg ObserverConfig) *Observer { cfg.MaxTrackedReplayBlocks = DefaultMaxTrackedReplayBlocks } return &Observer{ - cfg: cfg, - votes: make(map[VoteMessageKey]VoteMessage), - certificates: make(map[CertificateKey]Certificate), - replayBlocks: make(map[uint64]BlockID), - replayChecks: make(map[CertificateKey]certificateReplayCheck), + cfg: cfg, + votes: make(map[VoteMessageKey]VoteMessage), + certificates: make(map[CertificateKey]Certificate), + replayBlocks: make(map[uint64]BlockID), + replayChecks: make(map[CertificateKey]certificateReplayCheck), + pendingReplayCertificates: make(map[CertificateKey]BlockID), } } @@ -346,12 +353,16 @@ func (o *Observer) trackCertificateLocked(key CertificateKey, cert Certificate) return false } o.certificates[key] = cert + if block, ok := cert.Block(); ok && block.HasHash() { + o.pendingReplayCertificates[key] = block + } o.certOrder = append(o.certOrder, key) for len(o.certificates) > o.cfg.MaxTrackedCertificates { old := o.certOrder[0] o.certOrder = o.certOrder[1:] delete(o.certificates, old) delete(o.replayChecks, old) + delete(o.pendingReplayCertificates, old) } return true } @@ -376,12 +387,10 @@ func (o *Observer) checkReplayBlockCertificatesLocked(block BlockID) { if !block.HasHash() { return } - for key, cert := range o.certificates { - certBlock, ok := cert.Block() - if !ok || certBlock.Slot != block.Slot { - continue + for key, certBlock := range o.pendingReplayCertificates { + if certBlock.Slot == block.Slot { + o.checkCertificateReplayLocked(key, o.certificates[key]) } - o.checkCertificateReplayLocked(key, cert) } } @@ -417,6 +426,7 @@ func (o *Observer) checkCertificateReplayLocked(key CertificateKey, cert Certifi } } o.replayChecks[key] = check + delete(o.pendingReplayCertificates, key) } type certificateReplayPendingStats struct { @@ -430,30 +440,24 @@ type certificateReplayPendingStats struct { func (o *Observer) certificateReplayPendingStatsLocked() certificateReplayPendingStats { var stats certificateReplayPendingStats - for key, cert := range o.certificates { - if _, checked := o.replayChecks[key]; checked { - continue + for _, certBlock := range o.pendingReplayCertificates { + stats.count++ + if stats.oldestSlot == 0 || certBlock.Slot < stats.oldestSlot { + stats.oldestSlot = certBlock.Slot } - certBlock, ok := cert.Block() - if ok && certBlock.HasHash() { - stats.count++ - if stats.oldestSlot == 0 || certBlock.Slot < stats.oldestSlot { - stats.oldestSlot = certBlock.Slot - } - if certBlock.Slot > stats.newestSlot { - stats.newestSlot = certBlock.Slot - } - if o.oldestReplayBlockSlot != 0 && certBlock.Slot < o.oldestReplayBlockSlot { - stats.preWindow++ - } - if o.oldestReplayBlockSlot != 0 && - o.latestReplayBlockSlot != 0 && - certBlock.Slot >= o.oldestReplayBlockSlot && - certBlock.Slot <= o.latestReplayBlockSlot { - stats.mature++ - if stats.matureOldestSlot == 0 || certBlock.Slot < stats.matureOldestSlot { - stats.matureOldestSlot = certBlock.Slot - } + if certBlock.Slot > stats.newestSlot { + stats.newestSlot = certBlock.Slot + } + if o.oldestReplayBlockSlot != 0 && certBlock.Slot < o.oldestReplayBlockSlot { + stats.preWindow++ + } + if o.oldestReplayBlockSlot != 0 && + o.latestReplayBlockSlot != 0 && + certBlock.Slot >= o.oldestReplayBlockSlot && + certBlock.Slot <= o.latestReplayBlockSlot { + stats.mature++ + if stats.matureOldestSlot == 0 || certBlock.Slot < stats.matureOldestSlot { + stats.matureOldestSlot = certBlock.Slot } } } diff --git a/pkg/alpenglow/observer_bench_test.go b/pkg/alpenglow/observer_bench_test.go index 8ff14d395..a792015cd 100644 --- a/pkg/alpenglow/observer_bench_test.go +++ b/pkg/alpenglow/observer_bench_test.go @@ -1,6 +1,9 @@ package alpenglow -import "testing" +import ( + "fmt" + "testing" +) func BenchmarkObserverVoteWithRetainedCertificates(b *testing.B) { o := NewObserver() @@ -20,3 +23,35 @@ func BenchmarkObserverVoteWithRetainedCertificates(b *testing.B) { } } } + +// Model a full diagnostic history, with a small unresolved frontier or an +// entirely unresolved history as a worst case. These are observer-only costs: +// no transactions, BLS verification, or network traffic are included. +func BenchmarkObserverEmptyReplay(b *testing.B) { + for _, pending := range []int{0, 32, DefaultMaxTrackedCertificates} { + for _, skips := range []int{0, 4} { + b.Run(fmt.Sprintf("pending=%d/skips=%d", pending, skips), func(b *testing.B) { + o := NewObserver() + for slot := uint64(1); slot <= DefaultMaxTrackedCertificates; slot++ { + if slot <= uint64(DefaultMaxTrackedCertificates-pending) { + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: slot, Hash: testHash(1)}}) + } + _, err := o.ObserveCertificate(Certificate{Type: CertificateNotarize, Slot: slot, + BlockHash: testHash(1), IncludedStake: 80, TotalStake: 100}) + if err != nil { + b.Fatal(err) + } + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + base := uint64(DefaultMaxTrackedCertificates + 1 + i*(skips+1)) + for n := 0; n < skips; n++ { + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: base + uint64(n)}}) + } + o.ObserveReplayBlock(ReplayBlockObservation{Block: BlockID{Slot: base + uint64(skips), Hash: testHash(1)}}) + } + }) + } + } +} diff --git a/pkg/alpenglow/observer_stats_test.go b/pkg/alpenglow/observer_stats_test.go index 3000de923..bf81999de 100644 --- a/pkg/alpenglow/observer_stats_test.go +++ b/pkg/alpenglow/observer_stats_test.go @@ -42,7 +42,16 @@ func TestObserverPendingStatsMatchFullScan(t *testing.T) { } o.Snapshot() o.mu.RLock() - cached, scanned := o.pendingStats, o.certificateReplayPendingStatsLocked() + cached, scanned := o.pendingStats, observerPendingStatsFullScan(o) + pending := make(map[CertificateKey]BlockID) + for key, cert := range o.certificates { + if _, checked := o.replayChecks[key]; !checked { + if block, ok := cert.Block(); ok && block.HasHash() { + pending[key] = block + } + } + } + require.Equal(t, pending, o.pendingReplayCertificates, "operation %d", i) o.mu.RUnlock() require.Equal(t, scanned, cached, "operation %d", i) } @@ -83,3 +92,36 @@ func TestObserverConcurrentSnapshotsAndReconciliation(t *testing.T) { require.Equal(t, uint64(100), snapshot.CertificateReplayMatches) require.Zero(t, snapshot.CertificateReplayPending) } + +// Independent reference retains the original scan over every retained certificate. +func observerPendingStatsFullScan(o *Observer) certificateReplayPendingStats { + var stats certificateReplayPendingStats + for key, cert := range o.certificates { + if _, checked := o.replayChecks[key]; checked { + continue + } + certBlock, ok := cert.Block() + if ok && certBlock.HasHash() { + stats.count++ + if stats.oldestSlot == 0 || certBlock.Slot < stats.oldestSlot { + stats.oldestSlot = certBlock.Slot + } + if certBlock.Slot > stats.newestSlot { + stats.newestSlot = certBlock.Slot + } + if o.oldestReplayBlockSlot != 0 && certBlock.Slot < o.oldestReplayBlockSlot { + stats.preWindow++ + } + if o.oldestReplayBlockSlot != 0 && + o.latestReplayBlockSlot != 0 && + certBlock.Slot >= o.oldestReplayBlockSlot && + certBlock.Slot <= o.latestReplayBlockSlot { + stats.mature++ + if stats.matureOldestSlot == 0 || certBlock.Slot < stats.matureOldestSlot { + stats.matureOldestSlot = certBlock.Slot + } + } + } + } + return stats +} From efcb29427f230bd2dc7f07c13898f0d9540bc13a Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:32:17 -0500 Subject: [PATCH 08/13] review: clarify ownership and archive investigation artifacts --- .gitattributes | 7 - docs/certificate-processing-evidence.md | 14 + docs/certpool-offlock.md | 77 ++---- docs/certpool-point-reuse.md | 57 ++-- .../2026-09-14/0-before.txt | 9 - ...mpare-BenchmarkCertPoolConcurrentVotes.txt | 6 - ...are-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 - ...idate-BenchmarkCertPoolConcurrentVotes.txt | 6 - ...ate-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 - .../2026-09-14/1-stage1.txt | 9 - ...idate-BenchmarkCertPoolConcurrentVotes.txt | 6 - ...ate-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 - .../2026-09-14/2-stage1.txt | 9 - .../2026-09-14/3-before.txt | 9 - ...mpare-BenchmarkCertPoolConcurrentVotes.txt | 6 - ...are-BenchmarkCertPoolFoldVerifiedBatch.txt | 9 - .../2026-09-14/4-before.txt | 9 - .../2026-09-14/5-stage1.txt | 9 - .../2026-09-14/6-stage1.txt | 9 - .../2026-09-14/7-before.txt | 9 - .../2026-09-14/README.md | 1 - .../2026-09-14/stage1-benchmarks.json | 246 ------------------ .../pr-split-2026-09-15/certificate/README.md | 7 - .../certificate/certificate-tests.log | 1 - .../certificate/certificate-vet.log | 0 pkg/alpenglow/certpool.go | 47 ++-- pkg/alpenglow/certpool_bench_test.go | 4 +- 27 files changed, 80 insertions(+), 513 deletions(-) delete mode 100644 .gitattributes create mode 100644 docs/certificate-processing-evidence.md delete mode 100644 docs/results/certificate-processing/2026-09-14/0-before.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/1-stage1.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/2-stage1.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/3-before.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/4-before.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/5-stage1.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/6-stage1.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/7-before.txt delete mode 100644 docs/results/certificate-processing/2026-09-14/README.md delete mode 100644 docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json delete mode 100644 docs/results/pr-split-2026-09-15/certificate/README.md delete mode 100644 docs/results/pr-split-2026-09-15/certificate/certificate-tests.log delete mode 100644 docs/results/pr-split-2026-09-15/certificate/certificate-vet.log diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index d64360d9a..000000000 --- a/.gitattributes +++ /dev/null @@ -1,7 +0,0 @@ -# Keep raw benchmark evidence available without overwhelming the review. -# Retain raw Go CPU padding and test-framework space/tab indentation. -docs/results/**/*.json linguist-generated=true -docs/results/**/*.jsonl linguist-generated=true -docs/results/**/*.txt linguist-generated=true whitespace=-blank-at-eol -docs/results/**/*.log linguist-generated=true -whitespace -docs/results/**/*.tar.gz linguist-generated=true diff --git a/docs/certificate-processing-evidence.md b/docs/certificate-processing-evidence.md new file mode 100644 index 000000000..2f4827a74 --- /dev/null +++ b/docs/certificate-processing-evidence.md @@ -0,0 +1,14 @@ +# Certificate Processing: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/blob/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc) +(tag `review-evidence-20260916-certificate-processing`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/docs/certpool-offlock.md b/docs/certpool-offlock.md index 0b8fcaaa6..646d3e70b 100644 --- a/docs/certpool-offlock.md +++ b/docs/certpool-offlock.md @@ -10,30 +10,14 @@ Each completed verified batch publishes promptly, outside both mutexes. New arri Point reuse is included: aggregation uses already-verified signature points, failed batches subdivide parsed members, and the unused tally public-key aggregate is removed. Randomized coefficients, signature checks, stake thresholds, equivocation budgets and paired-vote disjointness remain. -## Measurements +## Lock ownership -Native AMD Ryzen 7 9700X, Go 1.26.4, with the validator and its normal continuous load running. Diagnostic processes used Nice 15; compilation used GOMAXPROCS 2. Diagnostic intervals are excluded from live comparisons. - -Point reuse alone, four alternating samples per version, median full pending-vote fold: - -| Batch size | Original | Point reuse | Time reduction | -| --- | ---: | ---: | ---: | -| 1 | 0.681 ms | 0.655 ms | 3.8% | -| 8 | 2.567 ms | 2.357 ms | 8.2% | -| 32 | 8.460 ms | 7.580 ms | 10.4% | -| 64 | 16.637 ms | 15.127 ms | 9.1% | - -The concurrent diagnostic uses eight producers submitting 64 votes, samples Snapshot latency, then flushes and checks that all 64 votes published exactly once. In two alternating samples per version, Snapshot p95 was 8.57–8.72 ms with point reuse alone and 0.008–0.038 ms with off-lock verification. Benchmark ns/op includes deliberately spaced sampling; it is not production throughput. - -The combined build deployed September 14 at 22:46 UTC. The 45-second live trace measured 36,133 normally completed incoming calls: initial mutex wait median 1.24us, p95 3.64us, p99 9.10us and maximum 0.369ms. None waited over 1ms, versus 11,742 in the earlier baseline. Different live windows and probe overhead are limitations; this is not an equivalent end-to-end voting speedup. Verification-gate and admission-condition waits are separate from the measured initial mutex acquisition. Two durable-floor calls waited at most 1.95us. - -Live voting/FAST results, exact deployment evidence and raw measurements are in the workspace at `mithril-run-20260911/certpool-offlock-20260914/RESULTS.md`. Native artifacts are under `/srv/mithril-certpool-offlock-20260914`. Four TPU workers, two shred workers, GOMAXPROCS 8, reserved vote history, previous live integrations, continuous load policy and signing/payer ledgers are preserved. - -## Validation - -New deterministic race tests park real work at the verification gate and cover concurrent admission, in-flight accounting, duplicate admission, capacity wakeup, pruning, eviction, changed bindings and reward-flush waiting. Existing invalid-share cancellation, malformed signature, aggregate-certificate, equivocation, fallback and publication tests remain. - -Native race tests passed for Alpenglow, consensus, replay and node integration. Peer isolation tests separately passed three native runs; an intermittent timeout had reproduced on the unchanged local baseline earlier. Native vet and the application build passed. The four deployed source/test files match local hashes. Instruction probes and nearby stack/register operations were checked against old/new disassembly, normalizing linker relocations. A live sanity capture observed 133 replays and 133 notarize events with no reservation exhaustion or admission rejection. +`verifyAndFoldTallyWithLockReleased` requires the pool lock on entry and returns +with it held, including early exits. It releases the lock during crypto and +revalidates slot and validator bindings after reacquiring it. The caller owns +that slot's processing marker. `finishSlotAndUnlock` consumes lock ownership: +it releases the lock, emits certificates and returns unlocked. Callers must not +pair it with a deferred unlock. ## Bounded multi-scalar aggregation @@ -53,36 +37,6 @@ coefficient. Entropy/aggregation errors retain the individual-verification fallback. Parsing, subgroup/infinity checks, failed-batch subdivision, publication and stake accounting are unchanged. -### Local component measurements - -Apple M4 Pro, Go 1.26.4, one caller, GOMAXPROCS=12. Three samples per version; -medians below compare the split branch before this change with bounded -multi-scalar aggregation. `BenchmarkCertPoolFoldVerifiedBatch` includes parsing, -pending-map setup, verification and tally folding; fixture signing is excluded. - -| Votes in fold | Scalar implementation | Bounded MultiExp | -| --- | ---: | ---: | -| 32 | 11.23 ms | 6.73 ms | -| 64 | 21.26 ms | 10.61 ms | - -A same-binary comparison of the weighted-pairing helpers found 2-vote batches -slower with MultiExp (1.49 → 2.04 ms) and 4-vote batches slower (2.01 → 2.44 ms). -Those sizes therefore retain the scalar implementation in production. At 64 -votes, allocated bytes per full fold increased from approximately 225 KB to -274 KB, although allocation count decreased from 1,204 to approximately 922. - -Run `go test ./pkg/alpenglow -run '^$' -bench '^BenchmarkCertPool(WeightedPairing|FoldVerifiedBatch)$' -benchmem -benchtime=500ms -count=3`. - -These are local component measurements; this aggregation change has not been -deployed. Native staging measurements follow below. The earlier deployment -measurements above describe the preceding implementation. -The full local Alpenglow race suite and vet pass. Additional coverage checks -invalid members at every position, wrong payloads, cancelling invalid shares -in larger batches, and subdivision across the scalar/MultiExp boundary. - -The 16-vote cutoff was also measured directly: weighted pairing improved from -5.28 ms to 3.80 ms locally and from 3.44 ms to 2.29 ms on Zen 5. - ### Native staging validation AMD Ryzen 7 9700X, Go 1.26.4, GOMAXPROCS=2, Nice 15, two-core CPU quota, with @@ -110,8 +64,15 @@ execution improved slightly. The final implementation therefore keeps the original scalar arithmetic whenever GOMAXPROCS is one. No worker-count or verification-admission changes accompany this optimization. -Final combined native race suites, targeted adversarial tests at GOMAXPROCS=1 -and 2, vet and the validator production build passed. Native source hashes -matched the complete local combined candidate. Live FAST, durable-root lag and -voting latency still need a deployment comparison; these staging results do -not establish those gains. +## Reproduce and validate + +Run `go test -race ./pkg/alpenglow` for concurrency, pending-budget, stale-binding, +invalid-share, equivocation and publication-barrier coverage. +Run `go test ./pkg/alpenglow -run '^$' -bench '^BenchmarkCertPool(WeightedPairing|FoldVerifiedBatch)$' -benchmem -benchtime=500ms -count=3` with the same fixture on each revision. + +The earlier concurrent 64-vote diagnostic reduced Snapshot p95 from +8.57–8.72 ms with point reuse alone to 0.008–0.038 ms with verification outside +the lock. This measures reader latency, not vote throughput. The diagnostic's +ns/op includes intentional sampling pauses. Historical component baselines, +live observations and full qualifications are preserved in the +[evidence archive](certificate-processing-evidence.md). diff --git a/docs/certpool-point-reuse.md b/docs/certpool-point-reuse.md index cb99f6008..cc2ac3a84 100644 --- a/docs/certpool-point-reuse.md +++ b/docs/certpool-point-reuse.md @@ -1,39 +1,18 @@ -# Reuse verified BLS signature points in the incoming vote pool - -This records the isolated first step. It was subsequently measured on Zen 5 and included in the [off-lock verification deployment](certpool-offlock.md); the local measurements and validation history below are retained. - -Incoming vote verification parsed each BLS signature, returned the original wire message, and parsed the same signature again when folding it into the tally. Failed aggregate checks recursively reparsed their subsets too. The tally also accumulated a public-key sum that was never read. - -The pool now returns the verified parsed members and uses their signature points directly for tally aggregation. Failed batches subdivide parsed members, and the individual-verification fallback returns verified parsed members as well. The unused tally public-key aggregate is removed; the randomized verifier's essential weighted public-key aggregate remains. - -This is the first, contained optimization from the September 14 certificate-pool investigation. The shared mutex, thresholds, publication ordering, pending limits, stake accounting, duplicate/equivocation checks, paired-vote disjointness, subgroup/infinity checks, and randomized coefficients remain in place. Moving verification outside the lock is a separate future change. Installed validator sets already cache parsed public keys. - -## Benchmark - -Local Apple M4 Pro, darwin/arm64, Go 1.26.4. `BenchmarkCertPoolFoldVerifiedBatch` exercises pending-map setup, verification, tally aggregation and accounting for valid same-payload batches, with installed public-key caches. Signing and fixture preparation are outside the timer. Each benchmark runs one goroutine with `-test.cpu=1` and `-test.benchtime=500ms`. - -The baseline is the pre-change production source at commit `0af2e094`, with the identical benchmark added. Separate before/after test binaries were built before measurement. Four measurements per version alternate in before/after/after/before order twice; the table shows medians. Tests and compilation finished before benchmarking. - -| Votes per batch | Before | After | Time reduction | Allocations before → after | -| --- | ---: | ---: | ---: | ---: | -| 1 | 0.985 ms | 0.939 ms | 4.7% | 80 → 73 | -| 8 | 3.845 ms | 3.606 ms | 6.2% | 222 → 208 | -| 32 | 12.724 ms | 11.089 ms | 12.8% | 679 → 641 | -| 64 | 25.290 ms | 21.449 ms | 15.2% | 1,263 → 1,193 | - -These are local component measurements, not Zen 5 or live FAST results. No validator restart/deployment, load-policy change, or live profiling was performed for this implementation. Large-batch time improves by several milliseconds locally, but this does not establish the reduction in lock waits or end-to-end voting latency. - -Raw samples, binary hashes and the runner are preserved in the workspace under `mithril-run-20260911/non-replay-latency-20260914/point-reuse/` and `benchmark-points.py` in its parent directory. - -## Validation - -Passed: - -* All targeted `TestCertPool` and `TestIndividuallyVerifiedBatch` tests, including three runs with race detection. -* New differential coverage checks retained points/messages against individual verification for valid batches, wrong-payload signatures in both recursive halves, malformed encodings, infinity and invalid ranks. The individual-verification fallback is checked separately. -* Existing cancellation-attack, invalid-candidate poisoning, aggregate-certificate verification, pruning, equivocation and reward-publication barrier coverage. -* Full `pkg/consensus` race suite. -* `pkg/alpenglow` race suite excluding `TestVotorBroadcasterIsolatesBlockedPeer`. -* `go vet ./pkg/alpenglow ./pkg/consensus`, `go build ./cmd/mithril`, and `git diff --check`. - -The unfiltered Alpenglow race suite hit an intermittent timeout in `TestVotorBroadcasterIsolatesBlockedPeer/reconnect`, at `peer_sender_test.go:209` waiting for the stalled connection to close. The same timeout reproduced with the unchanged certificate-pool source through a Go overlay. No transport code or timeout was changed to mask it. This pre-existing failure remains an explicit qualification limitation; the full suite is not reported as passing. +# Reuse verified BLS signature points + +Incoming verification returns parsed, verified members and reuses their signature +points when folding the tally. Failed aggregate checks subdivide parsed members +instead of reparsing each subset. Individual verification returns the same member +representation. Installed validator sets retain their parsed public keys. + +The unused tally public-key sum is removed. Randomized verification still builds +its required weighted public-key sum with independent full-field coefficients. +Subgroup/infinity checks, invalid-share rejection, duplicate/equivocation checks, +stake accounting and paired-vote disjointness remain enforced. + +Point ownership and message association are covered by differential tests against +individual verification, including malformed signatures, invalid ranks and wrong +payloads. See [pool concurrency and aggregation](certpool-offlock.md) for the +current lock contract, combined implementation and benchmark method. +The isolated point-reuse experiment is preserved in the +[historical evidence](certificate-processing-evidence.md). diff --git a/docs/results/certificate-processing/2026-09-14/0-before.txt b/docs/results/certificate-processing/2026-09-14/0-before.txt deleted file mode 100644 index 886c64f84..000000000 --- a/docs/results/certificate-processing/2026-09-14/0-before.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 894 663584 ns/op 10417 B/op 80 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 232 2555742 ns/op 33852 B/op 222 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 70 8424882 ns/op 120269 B/op 679 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 34 16426787 ns/op 232923 B/op 1263 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt deleted file mode 100644 index a9488ca64..000000000 --- a/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt +++ /dev/null @@ -1,6 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolConcurrentVotes-8 18 31244638 ns/op 9756702 snapshot-max-ns 8565800 snapshot-p95-ns 276815 B/op 1762 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt deleted file mode 100644 index d828970a0..000000000 --- a/docs/results/certificate-processing/2026-09-14/0-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 951 641175 ns/op 10537 B/op 80 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 259 2300836 ns/op 33003 B/op 215 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 80 7344967 ns/op 116161 B/op 648 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14097224 ns/op 224464 B/op 1200 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt deleted file mode 100644 index b90df1c00..000000000 --- a/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolConcurrentVotes.txt +++ /dev/null @@ -1,6 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolConcurrentVotes-8 34 17816437 ns/op 199885 snapshot-max-ns 38282 snapshot-p95-ns 255776 B/op 1579 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt deleted file mode 100644 index d0bb23c76..000000000 --- a/docs/results/certificate-processing/2026-09-14/1-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 943 636360 ns/op 10857 B/op 83 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 262 2291882 ns/op 33323 B/op 218 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 80 7409936 ns/op 116481 B/op 651 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14327493 ns/op 224784 B/op 1203 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/1-stage1.txt b/docs/results/certificate-processing/2026-09-14/1-stage1.txt deleted file mode 100644 index c6f4b9ae4..000000000 --- a/docs/results/certificate-processing/2026-09-14/1-stage1.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 915 660591 ns/op 9689 B/op 73 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 252 2363508 ns/op 32155 B/op 208 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 79 7789873 ns/op 115313 B/op 641 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 39 15996987 ns/op 223619 B/op 1193 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt deleted file mode 100644 index a46f8dcc7..000000000 --- a/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolConcurrentVotes.txt +++ /dev/null @@ -1,6 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolConcurrentVotes-8 34 16300179 ns/op 148197 snapshot-max-ns 8365 snapshot-p95-ns 255690 B/op 1575 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt deleted file mode 100644 index 960594bc4..000000000 --- a/docs/results/certificate-processing/2026-09-14/2-candidate-BenchmarkCertPoolFoldVerifiedBatch.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 924 702853 ns/op 10857 B/op 83 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 246 2295439 ns/op 33323 B/op 218 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 81 7358090 ns/op 116481 B/op 651 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14125492 ns/op 224784 B/op 1203 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/2-stage1.txt b/docs/results/certificate-processing/2026-09-14/2-stage1.txt deleted file mode 100644 index f3f370099..000000000 --- a/docs/results/certificate-processing/2026-09-14/2-stage1.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 872 652789 ns/op 9689 B/op 73 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 258 2472982 ns/op 32155 B/op 208 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 67 7768642 ns/op 115309 B/op 641 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 43 15097852 ns/op 223616 B/op 1193 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/3-before.txt b/docs/results/certificate-processing/2026-09-14/3-before.txt deleted file mode 100644 index c39fbbfe2..000000000 --- a/docs/results/certificate-processing/2026-09-14/3-before.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 856 662411 ns/op 10417 B/op 80 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 232 2661079 ns/op 33851 B/op 222 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 63 9372424 ns/op 120270 B/op 679 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 32 17757518 ns/op 232925 B/op 1263 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt b/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt deleted file mode 100644 index 09a8cf4ae..000000000 --- a/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolConcurrentVotes.txt +++ /dev/null @@ -1,6 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolConcurrentVotes-8 18 31545982 ns/op 9839357 snapshot-max-ns 8719558 snapshot-p95-ns 276423 B/op 1761 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt b/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt deleted file mode 100644 index 0e7dac2bf..000000000 --- a/docs/results/certificate-processing/2026-09-14/3-stage1-compare-BenchmarkCertPoolFoldVerifiedBatch.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 943 639930 ns/op 10537 B/op 80 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 259 2285913 ns/op 33003 B/op 215 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 79 7427797 ns/op 116161 B/op 648 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 40 14114964 ns/op 224466 B/op 1200 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/4-before.txt b/docs/results/certificate-processing/2026-09-14/4-before.txt deleted file mode 100644 index 78f039350..000000000 --- a/docs/results/certificate-processing/2026-09-14/4-before.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 846 731522 ns/op 10417 B/op 80 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 225 2575042 ns/op 33852 B/op 222 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 69 8494169 ns/op 120269 B/op 679 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 34 16556301 ns/op 232923 B/op 1263 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/5-stage1.txt b/docs/results/certificate-processing/2026-09-14/5-stage1.txt deleted file mode 100644 index 7d3bb27b2..000000000 --- a/docs/results/certificate-processing/2026-09-14/5-stage1.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 934 657515 ns/op 9688 B/op 73 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 258 2349599 ns/op 32155 B/op 208 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 79 7376201 ns/op 115313 B/op 641 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 42 14172786 ns/op 223616 B/op 1193 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/6-stage1.txt b/docs/results/certificate-processing/2026-09-14/6-stage1.txt deleted file mode 100644 index d2216cf06..000000000 --- a/docs/results/certificate-processing/2026-09-14/6-stage1.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 873 645598 ns/op 9689 B/op 73 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 260 2332481 ns/op 32155 B/op 208 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 82 7392025 ns/op 115312 B/op 641 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 40 15156833 ns/op 223618 B/op 1193 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/7-before.txt b/docs/results/certificate-processing/2026-09-14/7-before.txt deleted file mode 100644 index 45dec925d..000000000 --- a/docs/results/certificate-processing/2026-09-14/7-before.txt +++ /dev/null @@ -1,9 +0,0 @@ -goos: linux -goarch: amd64 -pkg: github.com/Overclock-Validator/mithril/pkg/alpenglow -cpu: AMD Ryzen 7 9700X 8-Core Processor -BenchmarkCertPoolFoldVerifiedBatch/votes=1 812 697862 ns/op 10417 B/op 80 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=8 235 2558921 ns/op 33851 B/op 222 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=32 72 8400940 ns/op 120268 B/op 679 allocs/op -BenchmarkCertPoolFoldVerifiedBatch/votes=64 34 16717873 ns/op 232923 B/op 1263 allocs/op -PASS diff --git a/docs/results/certificate-processing/2026-09-14/README.md b/docs/results/certificate-processing/2026-09-14/README.md deleted file mode 100644 index 96003cfbb..000000000 --- a/docs/results/certificate-processing/2026-09-14/README.md +++ /dev/null @@ -1 +0,0 @@ -Historical native Zen5 raw benchmarks supporting docs/certpool-offlock.md. before versus stage1 compares point reuse; stage1-compare versus candidate compares verification outside the lock. Concurrent ns/op includes deliberate sampling delays and is not throughput. Production pool source is preserved by this split; see the fresh split-branch validation separately. diff --git a/docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json b/docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json deleted file mode 100644 index 3d14f1300..000000000 --- a/docs/results/certificate-processing/2026-09-14/stage1-benchmarks.json +++ /dev/null @@ -1,246 +0,0 @@ -{ - "window": { - "start_utc": "2026-09-14T22:32:50.064174+00:00", - "end_utc": "2026-09-14T22:33:24.153378+00:00" - }, - "samples": { - "1": { - "before": [ - { - "ns": 663584.0, - "bytes": 10417.0, - "allocs": 80.0 - }, - { - "ns": 662411.0, - "bytes": 10417.0, - "allocs": 80.0 - }, - { - "ns": 731522.0, - "bytes": 10417.0, - "allocs": 80.0 - }, - { - "ns": 697862.0, - "bytes": 10417.0, - "allocs": 80.0 - } - ], - "stage1": [ - { - "ns": 660591.0, - "bytes": 9689.0, - "allocs": 73.0 - }, - { - "ns": 652789.0, - "bytes": 9689.0, - "allocs": 73.0 - }, - { - "ns": 657515.0, - "bytes": 9688.0, - "allocs": 73.0 - }, - { - "ns": 645598.0, - "bytes": 9689.0, - "allocs": 73.0 - } - ] - }, - "8": { - "before": [ - { - "ns": 2555742.0, - "bytes": 33852.0, - "allocs": 222.0 - }, - { - "ns": 2661079.0, - "bytes": 33851.0, - "allocs": 222.0 - }, - { - "ns": 2575042.0, - "bytes": 33852.0, - "allocs": 222.0 - }, - { - "ns": 2558921.0, - "bytes": 33851.0, - "allocs": 222.0 - } - ], - "stage1": [ - { - "ns": 2363508.0, - "bytes": 32155.0, - "allocs": 208.0 - }, - { - "ns": 2472982.0, - "bytes": 32155.0, - "allocs": 208.0 - }, - { - "ns": 2349599.0, - "bytes": 32155.0, - "allocs": 208.0 - }, - { - "ns": 2332481.0, - "bytes": 32155.0, - "allocs": 208.0 - } - ] - }, - "32": { - "before": [ - { - "ns": 8424882.0, - "bytes": 120269.0, - "allocs": 679.0 - }, - { - "ns": 9372424.0, - "bytes": 120270.0, - "allocs": 679.0 - }, - { - "ns": 8494169.0, - "bytes": 120269.0, - "allocs": 679.0 - }, - { - "ns": 8400940.0, - "bytes": 120268.0, - "allocs": 679.0 - } - ], - "stage1": [ - { - "ns": 7789873.0, - "bytes": 115313.0, - "allocs": 641.0 - }, - { - "ns": 7768642.0, - "bytes": 115309.0, - "allocs": 641.0 - }, - { - "ns": 7376201.0, - "bytes": 115313.0, - "allocs": 641.0 - }, - { - "ns": 7392025.0, - "bytes": 115312.0, - "allocs": 641.0 - } - ] - }, - "64": { - "before": [ - { - "ns": 16426787.0, - "bytes": 232923.0, - "allocs": 1263.0 - }, - { - "ns": 17757518.0, - "bytes": 232925.0, - "allocs": 1263.0 - }, - { - "ns": 16556301.0, - "bytes": 232923.0, - "allocs": 1263.0 - }, - { - "ns": 16717873.0, - "bytes": 232923.0, - "allocs": 1263.0 - } - ], - "stage1": [ - { - "ns": 15996987.0, - "bytes": 223619.0, - "allocs": 1193.0 - }, - { - "ns": 15097852.0, - "bytes": 223616.0, - "allocs": 1193.0 - }, - { - "ns": 14172786.0, - "bytes": 223616.0, - "allocs": 1193.0 - }, - { - "ns": 15156833.0, - "bytes": 223618.0, - "allocs": 1193.0 - } - ] - } - }, - "summary": { - "1": { - "before": { - "ns": 680723.0, - "bytes": 10417.0, - "allocs": 80.0 - }, - "stage1": { - "ns": 655152.0, - "bytes": 9689.0, - "allocs": 73.0 - }, - "time_reduction_percent": 3.7564471892384987 - }, - "8": { - "before": { - "ns": 2566981.5, - "bytes": 33851.5, - "allocs": 222.0 - }, - "stage1": { - "ns": 2356553.5, - "bytes": 32155.0, - "allocs": 208.0 - }, - "time_reduction_percent": 8.197487983454499 - }, - "32": { - "before": { - "ns": 8459525.5, - "bytes": 120269.0, - "allocs": 679.0 - }, - "stage1": { - "ns": 7580333.5, - "bytes": 115312.5, - "allocs": 641.0 - }, - "time_reduction_percent": 10.392923338312531 - }, - "64": { - "before": { - "ns": 16637087.0, - "bytes": 232923.0, - "allocs": 1263.0 - }, - "stage1": { - "ns": 15127342.5, - "bytes": 223617.0, - "allocs": 1193.0 - }, - "time_reduction_percent": 9.074572369550026 - } - } -} \ No newline at end of file diff --git a/docs/results/pr-split-2026-09-15/certificate/README.md b/docs/results/pr-split-2026-09-15/certificate/README.md deleted file mode 100644 index 2d5c32a92..000000000 --- a/docs/results/pr-split-2026-09-15/certificate/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Review branch validation, September 15 - -This PR is split from #279 plus the later working-tree improvements. The tested code commit before this documentation commit was `d7b19cac27e9b9fb54b249b58cf2883b9ad9ecc6`. Tests ran locally on Apple M4 Pro, Go1.26.4, GOMAXPROCS3 and package parallelism2. Logs beside this file are the fresh split-branch checks, not native measurements. Existing Zen5 benchmark documents retain their original baselines and scope. No validator restart or deployment occurred during this reorganization. - -Four independent branches start at current alpenglow-dev33dde405. Voting is based on the certificate-processing PR; leader packing is based on the streaming-preparation PR. Runtime changes and status-cache changes are independent. Shared CLI/configuration additions need an ordinary three-file merge reconciliation when combining leader packing and voting. A separate audit checkout reconciled these additions and matched the preserved full implementation exactly across Go sources, module files, TOML configuration and CI. - -The branch-specific race suites and vet passed. The combined audit has a separately documented pre-existing intermittent peer reconnect timeout; this is not reported as an entirely green combined race run. diff --git a/docs/results/pr-split-2026-09-15/certificate/certificate-tests.log b/docs/results/pr-split-2026-09-15/certificate/certificate-tests.log deleted file mode 100644 index 0d2a04a55..000000000 --- a/docs/results/pr-split-2026-09-15/certificate/certificate-tests.log +++ /dev/null @@ -1 +0,0 @@ -ok github.com/Overclock-Validator/mithril/pkg/alpenglow 8.507s diff --git a/docs/results/pr-split-2026-09-15/certificate/certificate-vet.log b/docs/results/pr-split-2026-09-15/certificate/certificate-vet.log deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index 671af66e7..c179ef68e 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -34,7 +34,7 @@ const maxUnverifiedCandidatesPerRank = 2 // facts. Ingest (AddVote) only shape-checks, bounds buffering, and parks the // vote — it NEVER mutates dedupe, equivocation, disjointness, or tally state // that assembly or fork choice depends on. Those mutate only AFTER the BLS -// signature verifies (foldTallyLocked). This prevents a bogus vote for a +// signature verifies (verifyAndFoldTallyWithLockReleased). This prevents a bogus vote for a // victim rank from suppressing that validator's real vote (dedupe poisoning) // or forging equivocation evidence against an honest validator. // @@ -285,7 +285,7 @@ func (p *CertPool) setForSlotLocked(slot uint64) *ValidatorSet { // AddVote ingests one raw (unverified) votor vote. It ONLY shape-checks, bounds // buffering, and parks the vote in its (type, hash) tally. It does NOT touch // dedupe/equivocation/disjointness state — those mutate only after the vote's -// signature verifies (foldTallyLocked). A malformed vote, a vote outside the +// signature verifies (verifyAndFoldTallyWithLockReleased). A malformed vote, a vote outside the // trusted slot window, or a vote past a memory bound is dropped. func (p *CertPool) AddVote(msg VoteMessage) { if msg.Vote.ValidateBasic() != nil || len(msg.Signature) != BLSSignatureSize { @@ -362,7 +362,7 @@ func (p *CertPool) AddVote(msg VoteMessage) { p.foldPendingRankLocked(slot, ps, msg.Rank, set) } if p.slots[slot] != ps { - p.finishSlotLocked(slot, ps, nil) + p.finishSlotAndUnlock(slot, ps, nil) return } candidates = tl.pending[msg.Rank] @@ -376,7 +376,7 @@ func (p *CertPool) AddVote(msg VoteMessage) { p.foldAllPendingLocked(slot, ps, set) } if p.slots[slot] != ps { - p.finishSlotLocked(slot, ps, nil) + p.finishSlotAndUnlock(slot, ps, nil) return } if p.totalPending >= p.cfg.MaxPendingVotesTotal { @@ -425,11 +425,11 @@ func (p *CertPool) AddVote(msg VoteMessage) { if forceFold { if set := p.setForSlotLocked(slot); set != nil { - p.foldTallyLocked(slot, ps, tl, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } } emits := p.drainSlotLocked(slot, ps, false) - p.finishSlotLocked(slot, ps, emits) + p.finishSlotAndUnlock(slot, ps, emits) } func (p *CertPool) admissionNeedsFoldLocked(ps *poolSlot, msg VoteMessage) bool { @@ -472,7 +472,7 @@ func (p *CertPool) drainSlotLocked(slot uint64, ps *poolSlot, flush bool) []Cert if set := p.setForSlotLocked(slot); set != nil { for key, tl := range ps.tallies { if key.Type == VoteTypeSkip || key.Type == VoteTypeNotarize { - p.foldTallyLocked(slot, ps, tl, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } } } @@ -487,9 +487,11 @@ func (p *CertPool) drainSlotLocked(slot uint64, ps *poolSlot, flush bool) []Cert return emits } -// finishSlotLocked takes the publication barrier before making the slot -// available to a flushing caller, then releases mu and emits certificates. -func (p *CertPool) finishSlotLocked(slot uint64, ps *poolSlot, emits []Certificate) uint64 { +// finishSlotAndUnlock requires p.mu held and returns with p.mu released. +// It takes the publication barrier before making the slot available to a +// flushing caller, then unlocks and emits certificates. Callers must not defer +// an unlock across this call. +func (p *CertPool) finishSlotAndUnlock(slot uint64, ps *poolSlot, emits []Certificate) uint64 { if p.slots[slot] != ps { emits = nil } @@ -526,7 +528,7 @@ func (p *CertPool) OnValidatorSetInstalled(epoch uint64) { } ps.processing = true emits := p.drainSlotLocked(slot, ps, false) - p.finishSlotLocked(slot, ps, emits) + p.finishSlotAndUnlock(slot, ps, emits) } } @@ -545,7 +547,7 @@ func (p *CertPool) FlushRewardVotes(slot uint64) { } ps.processing = true emits := p.drainSlotLocked(slot, ps, true) - target := p.finishSlotLocked(slot, ps, emits) + target := p.finishSlotAndUnlock(slot, ps, emits) // Includes publication by an owner that was verifying when flush arrived. p.waitForPublication(target) } @@ -779,11 +781,11 @@ func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) { switch tk.Type { case VoteTypeNotarize: if foldAllNotar || foldNotar[tk.Hash] { - p.foldTallyLocked(slot, ps, tl, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } case VoteTypeSkip: if foldSkip { - p.foldTallyLocked(slot, ps, tl, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } } } @@ -934,8 +936,8 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) []Certificate } // Candidate stake crossed: fold pending votes (one pairing per tally). - p.foldTallyLocked(slot, ps, base, set) - p.foldTallyLocked(slot, ps, fb, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, base, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, fb, set) if p.slots[slot] != ps { return nil } @@ -963,13 +965,18 @@ func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) []Certificate return emits } -// foldTallyLocked batch-verifies a tally's pending votes. All sign the same +// verifyAndFoldTallyWithLockReleased requires p.mu held on entry and returns +// with p.mu held on every path. The caller must own ps.processing. Verification +// temporarily releases p.mu; retained slot and validator bindings are rechecked +// after reacquiring it before any verified results are installed. +// +// It batch-verifies a tally's pending votes. All sign the same // payload, so randomized weighted pubkey/signature sums need one pairing; // failures bisect to isolate the bad votes (dropped + counted). Only AFTER a // vote verifies does it update the durable per-slot state — the vote-budget / // equivocation ledger (verifiedHash) and base↔fallback disjointness — so raw // votes can never poison those. -func (p *CertPool) foldTallyLocked(slot uint64, ps *poolSlot, tl *tally, set *ValidatorSet) { +func (p *CertPool) verifyAndFoldTallyWithLockReleased(slot uint64, ps *poolSlot, tl *tally, set *ValidatorSet) { if p.slots[slot] != ps || tl == nil || len(tl.pending) == 0 { return } @@ -1109,7 +1116,7 @@ func sameInstalledValidatorSet(a, b *ValidatorSet) bool { func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, set *ValidatorSet) { for _, tl := range ps.tallies { if len(tl.pending[rank]) != 0 { - p.foldTallyLocked(slot, ps, tl, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } } return @@ -1117,7 +1124,7 @@ func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, func (p *CertPool) foldAllPendingLocked(slot uint64, ps *poolSlot, set *ValidatorSet) { for _, tl := range ps.tallies { - p.foldTallyLocked(slot, ps, tl, set) + p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) } return } diff --git a/pkg/alpenglow/certpool_bench_test.go b/pkg/alpenglow/certpool_bench_test.go index 8b29fcb37..a6864f0af 100644 --- a/pkg/alpenglow/certpool_bench_test.go +++ b/pkg/alpenglow/certpool_bench_test.go @@ -25,7 +25,7 @@ func BenchmarkCertPoolFoldVerifiedBatch(b *testing.B) { pool := NewCertPool(DefaultCertPoolConfig(), verifier, nil) pool.SetEpochLookup(func(uint64) uint64 { return installed.Epoch }) tl := newTally() - ps := &poolSlot{verifiedHash: make(map[voteDedupKey][]solana.Hash), pendingByRank: make(map[uint16]int)} + ps := &poolSlot{processing: true, verifiedHash: make(map[voteDedupKey][]solana.Hash), pendingByRank: make(map[uint16]int)} pool.slots[vote.Slot] = ps for _, msg := range batch { tl.pending[msg.Rank] = map[[sha256.Size]byte]VoteMessage{sha256.Sum256(msg.Signature): msg} @@ -34,7 +34,7 @@ func BenchmarkCertPoolFoldVerifiedBatch(b *testing.B) { pool.totalPending++ } pool.mu.Lock() - pool.foldTallyLocked(vote.Slot, ps, tl, &installed) + pool.verifyAndFoldTallyWithLockReleased(vote.Slot, ps, tl, &installed) pool.mu.Unlock() if len(tl.verified) != size || tl.stake != uint64(size) || pool.totalPending != 0 { b.Fatal("incomplete verified fold") From f25346577fb9733bc0bb7ceb8d40a50e65dd4123 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 15 Sep 2026 19:39:57 -0500 Subject: [PATCH 09/13] docs: validate archive links and formatting --- docs/certificate-processing-evidence.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/certificate-processing-evidence.md b/docs/certificate-processing-evidence.md index 2f4827a74..c7b520914 100644 --- a/docs/certificate-processing-evidence.md +++ b/docs/certificate-processing-evidence.md @@ -2,7 +2,7 @@ The maintained subsystem documentation and reusable Go benchmarks describe the implementation and reproduction method. Historical raw results and session -notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/blob/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc) +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc) (tag `review-evidence-20260916-certificate-processing`). They are omitted from this proposed merge. [Historical result files](https://github.com/Overclock-Validator/mithril/tree/72514abc5a2a98d2a2823fe92f0bfbbeedbcc9bc/docs/results) From 3f074edd24d4ffc94c4a70e2d41d997aaf9ece61 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:02:51 -0500 Subject: [PATCH 10/13] alpenglow: prioritize reward flushes and reuse vote dedupe keys --- pkg/alpenglow/certpool.go | 66 ++++++++++++++++----- pkg/alpenglow/certpool_concurrency_test.go | 68 ++++++++++++++++++++++ 2 files changed, 121 insertions(+), 13 deletions(-) diff --git a/pkg/alpenglow/certpool.go b/pkg/alpenglow/certpool.go index c179ef68e..33483cc10 100644 --- a/pkg/alpenglow/certpool.go +++ b/pkg/alpenglow/certpool.go @@ -138,10 +138,13 @@ type tallyKey struct { type poolSlot struct { // One caller owns folding for a slot while other callers may buffer votes. - // Both fields, like the maps below, are protected by CertPool.mu. + // All fields, like the maps below, are protected by CertPool.mu. processing bool - dirty bool - tallies map[tallyKey]*tally + // Queued and active reward flushes take precedence over new arrivals/owners. + // A count preserves priority when multiple footer builders overlap. + flushWaiting int + dirty bool + tallies map[tallyKey]*tally // verifiedHash tracks the block hashes a rank has cast VERIFIED votes for, // per (rank, type), for equivocation/vote-budget enforcement. Populated only // after signature verification — never from raw ingest — so a bogus vote can @@ -292,6 +295,7 @@ func (p *CertPool) AddVote(msg VoteMessage) { return } slot := msg.Vote.Slot + sigKey := sha256.Sum256(msg.Signature) p.mu.Lock() var ps *poolSlot @@ -328,7 +332,8 @@ func (p *CertPool) AddVote(msg VoteMessage) { // Ordinary arrivals can join the bounded pending maps during BLS work. // Quota pressure and competing signatures still wait for authentication // before admission, preserving the first-packet-poisoning protections. - if ps.processing && p.admissionNeedsFoldLocked(ps, msg) { + // A queued reward flush also stops new arrivals from extending its drain. + if ps.flushWaiting > 0 || (ps.processing && p.admissionNeedsFoldLocked(ps, msg, sigKey)) { p.workCond.Wait() continue } @@ -351,7 +356,6 @@ func (p *CertPool) AddVote(msg VoteMessage) { // one. Different block hashes remain separate tallies. forceFold := false if _, done := tl.verified[msg.Rank]; !done { - sigKey := sha256.Sum256(msg.Signature) candidates := tl.pending[msg.Rank] _, duplicate := candidates[sigKey] if !duplicate && ps.pendingByRank[msg.Rank] >= p.cfg.MaxPendingVotesPerRankSlot { @@ -432,14 +436,14 @@ func (p *CertPool) AddVote(msg VoteMessage) { p.finishSlotAndUnlock(slot, ps, emits) } -func (p *CertPool) admissionNeedsFoldLocked(ps *poolSlot, msg VoteMessage) bool { +func (p *CertPool) admissionNeedsFoldLocked(ps *poolSlot, msg VoteMessage, sigKey [sha256.Size]byte) bool { tl := ps.tallies[tallyKey{Type: msg.Vote.Type, Hash: msg.Vote.BlockHash}] if tl != nil { if _, done := tl.verified[msg.Rank]; done { return false } candidates := tl.pending[msg.Rank] - if _, duplicate := candidates[sha256.Sum256(msg.Signature)]; duplicate { + if _, duplicate := candidates[sigKey]; duplicate { return false } if len(candidates) > 0 { @@ -455,7 +459,7 @@ func (p *CertPool) admissionNeedsFoldLocked(ps *poolSlot, msg VoteMessage) bool func (p *CertPool) waitForSlotLocked(slot uint64) *poolSlot { for { ps := p.slots[slot] - if ps == nil || !ps.processing { + if ps == nil || (!ps.processing && ps.flushWaiting == 0) { return ps } p.workCond.Wait() @@ -464,6 +468,8 @@ func (p *CertPool) waitForSlotLocked(slot uint64) *poolSlot { // drainSlotLocked catches arrivals buffered while the owner was outside mu. // Sub-threshold votes remain lazy except when preparing a reward footer. +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. func (p *CertPool) drainSlotLocked(slot uint64, ps *poolSlot, flush bool) []Certificate { var emits []Certificate for p.slots[slot] == ps { @@ -535,10 +541,28 @@ func (p *CertPool) OnValidatorSetInstalled(epoch uint64) { // FlushRewardVotes batch-verifies every pending plain skip/notarize vote for a // reward slot. Normal consensus verification stays lazy, but block production // calls this just before building the slot+8 footer so valid below-threshold -// votes are not omitted from reward certificates. +// votes are not omitted from reward certificates. Queued flushes take precedence +// over new slot owners and arrivals; an existing owner finishes first. Pruning +// can invalidate that generation, in which case the current slot is rechecked. func (p *CertPool) FlushRewardVotes(slot uint64) { p.mu.Lock() - ps := p.waitForSlotLocked(slot) + var ps *poolSlot + for { + ps = p.slots[slot] + if ps == nil { + break + } + ps.flushWaiting++ + for p.slots[slot] == ps && ps.processing { + p.workCond.Wait() + } + if p.slots[slot] == ps { + break + } + // Pruning or a binding change replaced this generation while waiting. + ps.flushWaiting-- + p.workCond.Broadcast() + } if ps == nil { target := p.publicationTargetLocked() p.mu.Unlock() @@ -547,6 +571,7 @@ func (p *CertPool) FlushRewardVotes(slot uint64) { } ps.processing = true emits := p.drainSlotLocked(slot, ps, true) + ps.flushWaiting-- target := p.finishSlotAndUnlock(slot, ps, emits) // Includes publication by an owner that was verifying when flush arrived. p.waitForPublication(target) @@ -734,6 +759,8 @@ func meets(f Fraction, stake, total uint64) bool { // implementation (Agave) would. This is the ONLY fold policy: one fork-choice // behavior for observer and voting nodes alike; sub-trigger tallies still // cost nothing. +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. func (p *CertPool) maybeFoldTriggersLocked(slot uint64, ps *poolSlot) { if p.slots[slot] != ps { return @@ -892,6 +919,8 @@ func targetsForSlot(ps *poolSlot) []certTarget { // maybeAssembleLocked checks every assemblable target for the slot: folds // pending votes (batch verification) once candidate stake crosses the // threshold, and returns any newly assembled certificates for emission. +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. func (p *CertPool) maybeAssembleLocked(slot uint64, ps *poolSlot) []Certificate { if p.slots[slot] != ps { return nil @@ -981,11 +1010,18 @@ func (p *CertPool) verifyAndFoldTallyWithLockReleased(slot uint64, ps *poolSlot, return } batch := make([]VoteMessage, 0, pendingCandidateCount(tl)) + // Reuse admission keys without hashing each signature again on removal. + var keyStorage [64][sha256.Size]byte + keys := keyStorage[:0] + if cap(batch) > len(keyStorage) { + keys = make([][sha256.Size]byte, 0, cap(batch)) + } for rank, candidates := range tl.pending { count := len(candidates) if int(rank) < len(set.Validators) { - for _, msg := range candidates { + for key, msg := range candidates { batch = append(batch, msg) + keys = append(keys, key) } continue } else { @@ -1026,9 +1062,9 @@ func (p *CertPool) verifyAndFoldTallyWithLockReleased(slot uint64, ps *poolSlot, p.workCond.Broadcast() return } - for _, msg := range batch { + for i, msg := range batch { candidates := tl.pending[msg.Rank] - delete(candidates, sha256.Sum256(msg.Signature)) + delete(candidates, keys[i]) if len(candidates) == 0 { delete(tl.pending, msg.Rank) } @@ -1113,6 +1149,8 @@ func sameInstalledValidatorSet(a, b *ValidatorSet) bool { len(a.parsedPubkeys) == len(b.parsedPubkeys) && &a.parsedPubkeys[0] == &b.parsedPubkeys[0] } +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, set *ValidatorSet) { for _, tl := range ps.tallies { if len(tl.pending[rank]) != 0 { @@ -1122,6 +1160,8 @@ func (p *CertPool) foldPendingRankLocked(slot uint64, ps *poolSlot, rank uint16, return } +// Requires and returns with p.mu held, but may release it while folding; +// ps may have been removed or replaced on return. The caller owns ps.processing. func (p *CertPool) foldAllPendingLocked(slot uint64, ps *poolSlot, set *ValidatorSet) { for _, tl := range ps.tallies { p.verifyAndFoldTallyWithLockReleased(slot, ps, tl, set) diff --git a/pkg/alpenglow/certpool_concurrency_test.go b/pkg/alpenglow/certpool_concurrency_test.go index b1d4bc253..a97c9eea9 100644 --- a/pkg/alpenglow/certpool_concurrency_test.go +++ b/pkg/alpenglow/certpool_concurrency_test.go @@ -1,6 +1,7 @@ package alpenglow import ( + "fmt" "sync" "testing" "time" @@ -182,3 +183,70 @@ func TestCertPoolOffLockEvictionDoesNotResurrectSlot(t *testing.T) { t.Fatal("evicted work displaced or corrupted the nearer slot") } } + +func TestCertPoolRewardFlushPriority(t *testing.T) { + for _, prune := range []bool{false, true} { + t.Run(fmt.Sprintf("prune=%t", prune), func(t *testing.T) { + pool, _, keys, _ := newTestPool(t) + vote := NewSkipVote(500) + // A below-threshold vote must be published by the reward flush. + pool.AddVote(VoteMessage{Vote: vote, Rank: 4, Signature: signTestVote(t, vote, keys[4])}) + pool.mu.Lock() + ps := pool.slots[vote.Slot] + ps.processing = true // Hold ownership until both flushes are queued. + pool.mu.Unlock() + flush1 := certPoolAsync(func() { pool.FlushRewardVotes(vote.Slot) }) + flush2 := certPoolAsync(func() { pool.FlushRewardVotes(vote.Slot) }) + deadline := time.Now().Add(2 * time.Second) + for { + pool.mu.Lock() + waiting := ps.flushWaiting + pool.mu.Unlock() + if waiting == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("flushes did not register") + } + time.Sleep(time.Millisecond) + } + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + t.Cleanup(func() { once.Do(func() { close(release) }) }) + pool.SetVerifiedVoteSink(func(v VerifiedVote) { + if v.Message.Rank == 4 { + close(entered) + <-release + } + }) + msg := VoteMessage{Vote: vote, Rank: 0, Signature: signTestVote(t, vote, keys[0])} + arrival := certPoolAsync(func() { pool.AddVote(msg) }) + if prune { + pool.ObserveFloor(vote.Slot) + } else { + pool.mu.Lock() + ps.processing = false + pool.workCond.Broadcast() + pool.mu.Unlock() + waitCertPoolCall(t, entered) + if got := pool.Snapshot().VotesAccepted; got != 1 { + t.Fatalf("new arrival overtook reward flush: accepted=%d", got) + } + select { + case <-arrival: + t.Fatal("arrival escaped active flush") + default: + } + } + once.Do(func() { close(release) }) + waitCertPoolCall(t, flush1) + waitCertPoolCall(t, flush2) + waitCertPoolCall(t, arrival) + pool.mu.Lock() + defer pool.mu.Unlock() + if ps.flushWaiting != 0 { + t.Fatalf("leaked flush waiters: %d", ps.flushWaiting) + } + }) + } +} From 3973850c69d409dbe21c1fd4a06d92c5dc8f4296 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Sun, 13 Sep 2026 12:17:19 -0500 Subject: [PATCH 11/13] Fix Alpenglow Votor certificate compatibility with Firedancer --- pkg/alpenglow/testdata/README.md | 9 ++++ .../testdata/agave_votor_certificate.der | Bin 0 -> 249 bytes pkg/alpenglow/tls_identity.go | 51 +++++++++++------- pkg/alpenglow/tls_identity_test.go | 46 ++++++++++++++++ 4 files changed, 87 insertions(+), 19 deletions(-) create mode 100644 pkg/alpenglow/testdata/agave_votor_certificate.der diff --git a/pkg/alpenglow/testdata/README.md b/pkg/alpenglow/testdata/README.md index 92b75d4eb..10997bf71 100644 --- a/pkg/alpenglow/testdata/README.md +++ b/pkg/alpenglow/testdata/README.md @@ -12,3 +12,12 @@ The v4.3 vote wire message intentionally excludes validator rank and stake; the authenticated Votor transport identity supplies those values after decode. The shred version is the final little-endian `u16`. Certificate bitmap vectors use wincode's default bincode-compatible little-endian `u64` length. + +`agave_votor_certificate.der` is the 249-byte certificate constructed by +`anza-xyz/agave` commit `8fe3f1201abc5b0244540aed0c7bf8c6bcafb3f5`, +`tls-utils/src/tls_certificates.rs::new_dummy_x509_certificate`, with public +key bytes `00..1f` at offsets 100–131. It also matches Firedancer commit +`de039cd7fc9f4714782ec7e3d47db3728903abc6`, +`src/ballet/x509/fd_x509_mock.c::fd_x509_mock_pubkey_v2`. Its fixed dummy +X.509 signature is intentional; TLS CertificateVerify proves possession of +the identity key. This fixture contains no private key. diff --git a/pkg/alpenglow/testdata/agave_votor_certificate.der b/pkg/alpenglow/testdata/agave_votor_certificate.der new file mode 100644 index 0000000000000000000000000000000000000000..e23bc0bfbe3e299e5b8ed04f44aad740f841d225 GIT binary patch literal 249 zcmXqL{ASR&ase|FBNGz`BNQ00vN3C?78r;biWms7F^94+^Kb{}=OpGOD&*y-q#7uQ z^O_qN7y=;}L`m?Q7+9Ji2^cUKXh98OR%BpcWMXDvWn<^yMC+6cQE@6%&_` zl#-T_m6KnrX`pT(4zx#Bkdg5}3$Fop6K76-a$-(KesPHb4@g27B*6qU7UD8yM~43t F0syv*UupmV literal 0 HcmV?d00001 diff --git a/pkg/alpenglow/tls_identity.go b/pkg/alpenglow/tls_identity.go index 1e8b91d91..3c18e0cfd 100644 --- a/pkg/alpenglow/tls_identity.go +++ b/pkg/alpenglow/tls_identity.go @@ -5,9 +5,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "fmt" - "math/big" "time" "github.com/gagliardetto/solana-go" @@ -27,6 +25,36 @@ func newVotorQUICConfig() *quic.Config { } } +// votorCertificateTemplate is Agave's dummy X.509 certificate from +// tls-utils/src/tls_certificates.rs (8fe3f1201abc5b0244540aed0c7bf8c6bcafb3f5). +// Firedancer's fd_x509_mock_pubkey_v2 requires this exact encoding except for +// the 32-byte SubjectPublicKeyInfo at offset 100. The X.509 signature is +// deliberately invalid: TLS 1.3 CertificateVerify authenticates the identity. +// Keep the template unchanged and copy it before inserting a public key. +var votorCertificateTemplate = [...]byte{ + 0x30, 0x81, 0xf6, 0x30, 0x81, 0xa9, 0xa0, 0x03, 0x02, 0x01, 0x02, 0x02, + 0x08, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x30, 0x05, 0x06, + 0x03, 0x2b, 0x65, 0x70, 0x30, 0x16, 0x31, 0x14, 0x30, 0x12, 0x06, 0x03, + 0x55, 0x04, 0x03, 0x0c, 0x0b, 0x53, 0x6f, 0x6c, 0x61, 0x6e, 0x61, 0x20, + 0x6e, 0x6f, 0x64, 0x65, 0x30, 0x20, 0x17, 0x0d, 0x37, 0x30, 0x30, 0x31, + 0x30, 0x31, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x5a, 0x18, 0x0f, 0x34, + 0x30, 0x39, 0x36, 0x30, 0x31, 0x30, 0x31, 0x30, 0x30, 0x30, 0x30, 0x30, + 0x30, 0x5a, 0x30, 0x00, 0x30, 0x2a, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, + 0x70, 0x03, 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xa3, 0x29, 0x30, 0x27, 0x30, 0x17, 0x06, 0x03, 0x55, 0x1d, 0x11, 0x01, + 0x01, 0xff, 0x04, 0x0d, 0x30, 0x0b, 0x82, 0x09, 0x6c, 0x6f, 0x63, 0x61, + 0x6c, 0x68, 0x6f, 0x73, 0x74, 0x30, 0x0c, 0x06, 0x03, 0x55, 0x1d, 0x13, + 0x01, 0x01, 0xff, 0x04, 0x02, 0x30, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, + 0x65, 0x70, 0x03, 0x41, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +} + // newVotorQUICCertificate creates the single-certificate Ed25519 identity // chain used by the Agave Votor transport. Peers recover the validator identity // directly from the leaf certificate's SubjectPublicKeyInfo; TLS 1.3's @@ -47,23 +75,8 @@ func newVotorQUICCertificate(identity ed25519.PrivateKey) (tls.Certificate, erro pub = priv.Public().(ed25519.PublicKey) } - template := &x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: "Mithril Alpenglow observer", - }, - NotBefore: time.Unix(0, 0), - NotAfter: time.Date(4096, 1, 1, 0, 0, 0, 0, time.UTC), - KeyUsage: x509.KeyUsageDigitalSignature, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, - DNSNames: []string{"localhost"}, - BasicConstraintsValid: true, - } - - certDER, err := x509.CreateCertificate(rand.Reader, template, template, pub, priv) - if err != nil { - return tls.Certificate{}, err - } + certDER := append([]byte(nil), votorCertificateTemplate[:]...) + copy(certDER[100:100+ed25519.PublicKeySize], pub) cert, err := x509.ParseCertificate(certDER) if err != nil { return tls.Certificate{}, err diff --git a/pkg/alpenglow/tls_identity_test.go b/pkg/alpenglow/tls_identity_test.go index 7efaa4125..cb591e0fa 100644 --- a/pkg/alpenglow/tls_identity_test.go +++ b/pkg/alpenglow/tls_identity_test.go @@ -1,15 +1,61 @@ package alpenglow import ( + "context" "crypto/ed25519" "crypto/tls" "crypto/x509" + "net" + "os" "testing" "time" "github.com/stretchr/testify/require" ) +func TestVotorCertificateMatchesAgaveAndFiredancerTemplate(t *testing.T) { + // Independent fixture from Agave's constructor, also accepted by + // Firedancer's fd_x509_mock_pubkey_v2 byte-pattern parser. + fixture, err := os.ReadFile("testdata/agave_votor_certificate.der") + require.NoError(t, err) + require.Len(t, fixture, 249) + for _, seed := range []byte{61, 62} { + identity := ed25519.NewKeyFromSeed(bytesOf(seed, ed25519.SeedSize)) + certificate, err := newVotorQUICCertificate(identity) + require.NoError(t, err) + expected := append([]byte(nil), fixture...) + copy(expected[100:132], identity.Public().(ed25519.PublicKey)) + require.Equal(t, expected, certificate.Certificate[0]) + require.Equal(t, identity.Public(), certificate.Leaf.PublicKey) + // Do not replace the dummy signature with a real one: Firedancer's + // parser matches it too. Authentication happens in CertificateVerify. + require.Error(t, certificate.Leaf.CheckSignature(certificate.Leaf.SignatureAlgorithm, + certificate.Leaf.RawTBSCertificate, certificate.Leaf.Signature)) + } +} + +func TestVotorCertificateStillRequiresIdentityKeyPossession(t *testing.T) { + identity := ed25519.NewKeyFromSeed(bytesOf(63, ed25519.SeedSize)) + certificate, err := newVotorQUICCertificate(identity) + require.NoError(t, err) + certificate.PrivateKey = ed25519.NewKeyFromSeed(bytesOf(64, ed25519.SeedSize)) + serverConn, clientConn := net.Pipe() + t.Cleanup(func() { _ = serverConn.Close(); _ = clientConn.Close() }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + server := tls.Server(serverConn, &tls.Config{ + Certificates: []tls.Certificate{certificate}, MinVersion: tls.VersionTLS13, + }) + client := tls.Client(clientConn, &tls.Config{ + InsecureSkipVerify: true, MinVersion: tls.VersionTLS13, + }) + serverDone := make(chan error, 1) + go func() { serverDone <- server.HandshakeContext(ctx) }() + // Skipping the dummy X.509 signature does not bypass CertificateVerify. + require.ErrorContains(t, client.HandshakeContext(ctx), "invalid signature") + require.Error(t, <-serverDone) +} + func TestVotorPeerIdentityRequiresOneEd25519Certificate(t *testing.T) { identity := ed25519.NewKeyFromSeed(bytesOf(61, ed25519.SeedSize)) certificate, err := newVotorQUICCertificate(identity) From 0a86a01cd6f2cf276690153503fb64ee263a62dc Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Thu, 24 Sep 2026 00:15:43 -0500 Subject: [PATCH 12/13] alpenglow: isolate bounded peer delivery from vote persistence Extract transport and diagnostics from b44bf9db and 2202851c; retain final reviewed peer-sender implementation without signing reservations or history changes. --- docs/votor-peer-isolation.md | 99 +++++++++ pkg/alpenglow/broadcaster.go | 113 +++++----- pkg/alpenglow/peer_sender.go | 181 +++++++++++++++ pkg/alpenglow/peer_sender_recovery_test.go | 136 ++++++++++++ pkg/alpenglow/peer_sender_test.go | 245 +++++++++++++++++++++ pkg/consensus/voter.go | 54 +++-- 6 files changed, 744 insertions(+), 84 deletions(-) create mode 100644 docs/votor-peer-isolation.md create mode 100644 pkg/alpenglow/peer_sender.go create mode 100644 pkg/alpenglow/peer_sender_recovery_test.go create mode 100644 pkg/alpenglow/peer_sender_test.go diff --git a/docs/votor-peer-isolation.md b/docs/votor-peer-isolation.md new file mode 100644 index 000000000..061b0f80a --- /dev/null +++ b/docs/votor-peer-isolation.md @@ -0,0 +1,99 @@ +# Votor outbound peer isolation + +A stalled QUIC peer could previously occupy all 32 shared send/connect workers. +quic-go's `SendDatagram` blocks when its 32-frame connection queue is full. +Repeated jobs for that peer could therefore prevent votes reaching healthy +peers, even while the broadcaster reported zero queue drops. + +Each authenticated connection now owns one sender and a FIFO queue of at most +256 encoded messages. The encoded payload is immutable and shared across peer +queues. The existing bounded worker pool handles connection attempts only; +`VotorBroadcasterConfig.Workers` controls that pool. A blocked connection cannot +consume another peer's sender or a connection worker. + +A watchdog checks every 100 ms whether the active datagram's peer-queue wait +plus its current `SendDatagram` duration has reached one second. Occasional QUIC +PTO probes can free queue entries without proving delivery; they no longer +restart this budget for an old backlog. Dequeue also retires a connection before +feeding an already-one-second-old entry into QUIC, even if sends keep completing +between watchdog ticks. The effective active-send bound is one second from +fanout enqueue, plus up to one watchdog interval and runtime scheduling delay. + +Retirement closes that connection, wakes a blocked sender, discards its queued +copies and requests a bounded reconnect. This is an operational limit on local +queueing, not a consensus validity deadline or a remote-delivery guarantee. It +adds no per-send timer or goroutine. An idle connection is not expired merely +because its previous send had a long queue delay. + +## Failure and ordering semantics + +- The global `Enqueue` contract is unchanged: rejecting a newly signed message + from the global queue returns an error to the voting engine. +- Per-peer fanout remains best effort. A full peer queue rejects that peer's + newest copy, increments both `PeerQueueDrops` and the existing + `MessagesDropped` counter, and continues sending to other peers. +- Messages queued on a failed, removed or replaced connection are discarded and + counted in `PeerQueueDiscarded`. They are not transferred to a new connection + or address. A datagram expired at dequeue is counted in `PeerQueueDiscarded`; + a failed `SendDatagram` call is counted in `PeerSendErrors`. The watchdog and + dequeue path claim retirement under the sender mutex, counting one timeout. +- Every sender exit requests a bounded, deduplicated reconnect. This covers + both a remote-close notification and closure detected while dequeuing. Peer + departure, shutdown and a healthy replacement suppress obsolete requests; + normal remote closure no longer relies on the periodic reconciliation tick. +- Messages for disconnected peers increment `PeerSendsSkipped`, as before. + This change adds no automatic application-level retransmission. +- A single sender preserves local enqueue order within its connection. QUIC + datagrams themselves still do not guarantee arrival or ordering. +- Shutdown cancels connection attempts, closes the connections outside the + broadcaster mutex, drains their queues, and waits for sender exit. + +Signing decisions, persisted vote history, durable slot reservations, and +certificate validation are unchanged. + +## Observability + +The voting log adds `peer_queue_drops`, `peer_queue_discarded`, +`peer_send_timeouts`, and `peer_queue_max_delay`. These counters/high-water marks +survive reconnects. `PeerQueueMaxDelay` measures time from fanout enqueue to +sender dequeue, including entries retired before reaching QUIC; it does not +include time blocked inside `SendDatagram`. + +Voting snapshots also expose `broadcast_peer_queues` with each current peer's +identity, address, queue depth, time in the active send, last/max queue delay, +and queue drops. Durations in the JSON snapshot use nanoseconds. Per-connection +statistics disappear when that connection is replaced; totals remain available. +Successful `PeerSends` only means quic-go accepted the datagram, not that a +remote validator received it. + +## Regression coverage + +`TestVotorBroadcasterIsolatesBlockedPeer` establishes two real loopback QUIC +connections, then blackholes one UDP path. It observes the blocked +`datagramQueue.Add` stack and requires a healthy-peer vote to arrive within +250 ms while the failed connection is still open. It also covers peer queue +overflow, watchdog reconnection, fresh traffic after reconnect, shutdown with a +blocked sender, peer departure, and replacement of a blocked peer's address. +Its connection-retirement deadline is measured from the blackhole, with explicit +scheduler slack; the healthy-peer latency assertion remains 250 ms. + +Deterministic regressions reproduce a fresh send following an old queue wait, +remote closure while idle and while dequeuing, and an already-aged queue entry. +The reconnect fixture has no reconciliation loop, so a timer cannot hide a missed +reconnect trigger. The pre-fix failures and fresh validation are retained under +[historical evidence](https://github.com/Overclock-Validator/mithril/blob/54b233ff0e27fb929644f7d53bf4a699cb590cd8/docs/results/review-fixes/2026-09-15). + +The signature-verification config template and its tests now belong to the +streaming branch, which reads those settings. This standalone voting branch +retains its base branch's supported `tuning.sigverify_backend` template. + +```sh +go test -race ./pkg/alpenglow ./pkg/consensus -count=1 -timeout=180s +go vet ./pkg/alpenglow ./pkg/consensus +go build ./cmd/mithril +``` + +This is a transport regression, not a prediction of FAST score improvement. +The live validator needs a separately validated integrated build and matching +probe addresses before deployment; the PR checkout does not include every +change in the currently enrolled validator binary. diff --git a/pkg/alpenglow/broadcaster.go b/pkg/alpenglow/broadcaster.go index bd8e3ee7c..557ea997d 100644 --- a/pkg/alpenglow/broadcaster.go +++ b/pkg/alpenglow/broadcaster.go @@ -18,7 +18,7 @@ import ( const ( defaultVotorBroadcastQueue = 1024 - defaultVotorSendWorkers = 32 + defaultVotorConnectWorkers = 32 defaultVotorPeerJobQueue = 16384 defaultVotorPeerRefreshInterval = time.Second ) @@ -35,7 +35,8 @@ type VotorBroadcasterConfig struct { ShredVersion uint16 Peers VotorPeerSource QueueSize int - Workers int + // Workers bounds concurrent connection attempts. Sends are isolated per connection. + Workers int } type VotorBroadcasterStats struct { @@ -50,28 +51,25 @@ type VotorBroadcasterStats struct { ConnectionAttempts uint64 ConnectionErrors uint64 ConnectionJobsDropped uint64 + PeerQueueDrops uint64 + PeerQueueDiscarded uint64 + PeerSendTimeouts uint64 + PeerQueueMaxDelay time.Duration + PeerQueues []VotorPeerQueueStats LastPeerSendError string LastPeerSendErrorAt time.Time LastConnectionError string LastConnectionErrorAt time.Time } -type votorPeerJobKind uint8 - -const ( - votorPeerJobConnect votorPeerJobKind = iota - votorPeerJobSend -) - type votorPeerJob struct { - kind votorPeerJobKind - peer VotorPeer - payload []byte + peer VotorPeer } type votorConnection struct { - addr string - conn *quic.Conn + addr string + conn *quic.Conn + sender *votorPeerSender } type votorDial struct { @@ -109,6 +107,10 @@ type VotorBroadcaster struct { connectionAttempts atomic.Uint64 connectionErrors atomic.Uint64 connectionJobsDropped atomic.Uint64 + peerQueueDrops atomic.Uint64 + peerQueueDiscarded atomic.Uint64 + peerSendTimeouts atomic.Uint64 + peerQueueMaxDelay atomic.Int64 } func NewVotorBroadcaster(cfg VotorBroadcasterConfig) (*VotorBroadcaster, error) { @@ -122,7 +124,7 @@ func NewVotorBroadcaster(cfg VotorBroadcasterConfig) (*VotorBroadcaster, error) cfg.QueueSize = defaultVotorBroadcastQueue } if cfg.Workers <= 0 { - cfg.Workers = defaultVotorSendWorkers + cfg.Workers = defaultVotorConnectWorkers } certificate, err := newVotorQUICCertificate(cfg.Identity) if err != nil { @@ -151,7 +153,7 @@ func NewVotorBroadcaster(cfg VotorBroadcasterConfig) (*VotorBroadcaster, error) } b.wg.Add(2 + cfg.Workers) for range cfg.Workers { - go b.sendLoop() + go b.connectLoop() } go b.broadcastLoop() // Populate the desired set and queue the first bounded preconnects before @@ -199,71 +201,43 @@ func (b *VotorBroadcaster) broadcastLoop() { b.recordSendError(VotorPeer{}, fmt.Errorf("encode Votor message: %w", err)) continue } - peers, skipped := b.connectedPeers() + senders, skipped := b.connectedSenders() b.sendsSkipped.Add(uint64(skipped)) - for _, peer := range peers { - job := votorPeerJob{kind: votorPeerJobSend, peer: peer, payload: payload} - select { - case b.jobs <- job: - case <-b.done: - return - default: - b.dropped.Add(1) - } + job := votorDatagram{payload: payload, queuedAt: time.Now()} + for _, sender := range senders { + sender.enqueue(job) } } } } -func (b *VotorBroadcaster) sendLoop() { +// Connection attempts never occupy a peer's sender or delay connected peers. +func (b *VotorBroadcaster) connectLoop() { defer b.wg.Done() for { select { case <-b.done: return case job := <-b.jobs: - switch job.kind { - case votorPeerJobConnect: - b.connectPeer(job.peer.Identity) - case votorPeerJobSend: - if err := b.send(job.peer, job.payload); err != nil { - b.recordSendError(job.peer, err) - } else { - b.sends.Add(1) - } - } + b.connectPeer(job.peer.Identity) } } } -func (b *VotorBroadcaster) send(peer VotorPeer, payload []byte) error { - conn, ok := b.establishedConnection(peer) - if !ok { - b.queueConnect(peer.Identity) - return fmt.Errorf("send Votor datagram to %s (%s): no established connection", peer.Identity, peer.Addr) - } - err := conn.SendDatagram(payload) - if err == nil { - return nil - } - var tooLarge *quic.DatagramTooLargeError - if !errors.As(err, &tooLarge) { - b.dropConnection(peer.Identity, conn) - b.queueConnect(peer.Identity) - } - return fmt.Errorf("send Votor datagram to %s (%s): %w", peer.Identity, peer.Addr, err) -} - func (b *VotorBroadcaster) peerReconcileLoop() { defer b.wg.Done() ticker := time.NewTicker(defaultVotorPeerRefreshInterval) defer ticker.Stop() + watchdog := time.NewTicker(votorSendWatchInterval) + defer watchdog.Stop() for { select { case <-b.done: return case <-ticker.C: b.reconcilePeers() + case now := <-watchdog.C: + b.expirePeerSends(now) } } } @@ -305,9 +279,9 @@ func (b *VotorBroadcaster) reconcilePeers() { } } -func (b *VotorBroadcaster) connectedPeers() ([]VotorPeer, int) { +func (b *VotorBroadcaster) connectedSenders() ([]*votorPeerSender, int) { b.connMu.Lock() - peers := make([]VotorPeer, 0, len(b.desired)) + peers := make([]*votorPeerSender, 0, len(b.desired)) skipped := 0 for identity, peer := range b.desired { existing, connected := b.conns[identity] @@ -315,8 +289,7 @@ func (b *VotorBroadcaster) connectedPeers() ([]VotorPeer, int) { skipped++ continue } - peer.Addr = cloneUDPAddr(peer.Addr) - peers = append(peers, peer) + peers = append(peers, existing.sender) } b.connMu.Unlock() return peers, skipped @@ -349,7 +322,7 @@ func (b *VotorBroadcaster) queueConnectLocked(identity solana.PublicKey) { if _, queued := b.connectQueued[identity]; queued || b.dialing[identity] != nil { return } - job := votorPeerJob{kind: votorPeerJobConnect, peer: peer} + job := votorPeerJob{peer: peer} select { case b.jobs <- job: b.connectQueued[identity] = struct{}{} @@ -476,7 +449,12 @@ func (b *VotorBroadcaster) connection(peer VotorPeer) (*quic.Conn, error) { return existing.conn, nil } stale := b.conns[peer.Identity].conn - b.conns[peer.Identity] = votorConnection{addr: addr, conn: conn} + sender := &votorPeerSender{b: b, peer: peer, conn: conn, queue: make(chan votorDatagram, defaultVotorPeerSendQueue), done: make(chan struct{})} + b.conns[peer.Identity] = votorConnection{addr: addr, conn: conn, sender: sender} + // Close takes connMu before waiting, so no sender can be added after it + // observes the closed flag and drains the connection set. + b.wg.Add(1) + go sender.run() b.connMu.Unlock() if stale != nil { _ = stale.CloseWithError(0, "Votor peer address changed") @@ -499,12 +477,14 @@ func (b *VotorBroadcaster) Stats() VotorBroadcasterStats { } b.connMu.Lock() connections := 0 + peerQueues := make([]VotorPeerQueueStats, 0, len(b.conns)) for identity, existing := range b.conns { if existing.conn.Context().Err() != nil { delete(b.conns, identity) continue } connections++ + peerQueues = append(peerQueues, existing.sender.stats()) } desiredPeers := len(b.desired) pendingConnections := len(b.connectQueued) @@ -525,6 +505,11 @@ func (b *VotorBroadcaster) Stats() VotorBroadcasterStats { ConnectionAttempts: b.connectionAttempts.Load(), ConnectionErrors: b.connectionErrors.Load(), ConnectionJobsDropped: b.connectionJobsDropped.Load(), + PeerQueueDrops: b.peerQueueDrops.Load(), + PeerQueueDiscarded: b.peerQueueDiscarded.Load(), + PeerSendTimeouts: b.peerSendTimeouts.Load(), + PeerQueueMaxDelay: time.Duration(b.peerQueueMaxDelay.Load()), + PeerQueues: peerQueues, LastPeerSendError: lastSendError, LastPeerSendErrorAt: lastSendErrorAt, LastConnectionError: lastConnectionError, @@ -542,11 +527,15 @@ func (b *VotorBroadcaster) Close() error { close(b.done) b.connMu.Lock() clear(b.desired) + stale := make([]*quic.Conn, 0, len(b.conns)) for identity, existing := range b.conns { - _ = existing.conn.CloseWithError(0, "Votor broadcaster closed") + stale = append(stale, existing.conn) delete(b.conns, identity) } b.connMu.Unlock() + for _, conn := range stale { + _ = conn.CloseWithError(0, "Votor broadcaster closed") + } b.wg.Wait() }) return nil diff --git a/pkg/alpenglow/peer_sender.go b/pkg/alpenglow/peer_sender.go new file mode 100644 index 000000000..73c9176f6 --- /dev/null +++ b/pkg/alpenglow/peer_sender.go @@ -0,0 +1,181 @@ +package alpenglow + +import ( + "errors" + "sync" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/quic-go/quic-go" +) + +const ( + defaultVotorPeerSendQueue = 256 + votorSendTimeout = time.Second + votorSendWatchInterval = 100 * time.Millisecond +) + +// VotorPeerQueueStats describes local queueing, not remote delivery. Counters +// here cover the current connection; broadcaster totals survive reconnects. +type VotorPeerQueueStats struct { + Identity solana.PublicKey `json:"identity"` + Address string `json:"address"` + Queued int `json:"queued"` + SendingFor time.Duration `json:"sending_for_ns"` + LastQueueDelay time.Duration `json:"last_queue_delay_ns"` + MaxQueueDelay time.Duration `json:"max_queue_delay_ns"` + QueueDrops uint64 `json:"queue_drops"` +} + +type votorDatagram struct { + payload []byte // immutable, shared across the peer queues + queuedAt time.Time +} + +// Each authenticated connection owns one sender. No mutex is held while +// SendDatagram blocks on quic-go's bounded datagram queue. +type votorPeerSender struct { + b *VotorBroadcaster + peer VotorPeer + conn *quic.Conn + queue chan votorDatagram + done chan struct{} + + mu sync.Mutex + closed bool + sendingSince time.Time + lastQueueDelay time.Duration + maxQueueDelay time.Duration + queueDrops uint64 +} + +func (s *votorPeerSender) enqueue(job votorDatagram) { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed || s.conn.Context().Err() != nil { + s.b.sendsSkipped.Add(1) + return + } + select { + case s.queue <- job: + default: + // A full queue rejects only this peer's copy. As before, fanout is + // best effort; the global Enqueue error contract is unchanged. + s.queueDrops++ + s.b.peerQueueDrops.Add(1) + s.b.dropped.Add(1) + } +} + +func (s *votorPeerSender) run() { + defer s.b.wg.Done() + defer close(s.done) + // Cover both remote-close select and close detected after dequeuing a job. + // The queue is bounded/deduplicated; shutdown, departure and a healthy + // replacement connection suppress obsolete reconnect requests. + defer s.b.queueConnect(s.peer.Identity) + defer func() { + s.mu.Lock() + s.closed = true + // Old-connection work is not replayed onto a new address/connection. + // Account for every queued copy discarded on failure or shutdown. + for { + select { + case <-s.queue: + s.b.peerQueueDiscarded.Add(1) + default: + s.mu.Unlock() + return + } + } + }() + for { + select { + case <-s.b.done: + return + case <-s.conn.Context().Done(): + return + case job := <-s.queue: + s.mu.Lock() + if s.closed || s.conn.Context().Err() != nil || s.b.closed.Load() { + s.mu.Unlock() + s.b.peerQueueDiscarded.Add(1) + return + } + now := time.Now() + s.sendingSince = now + s.lastQueueDelay = now.Sub(job.queuedAt) + s.maxQueueDelay = max(s.maxQueueDelay, s.lastQueueDelay) + for old := s.b.peerQueueMaxDelay.Load(); int64(s.lastQueueDelay) > old; old = s.b.peerQueueMaxDelay.Load() { + if s.b.peerQueueMaxDelay.CompareAndSwap(old, int64(s.lastQueueDelay)) { + break + } + } + if s.lastQueueDelay >= votorSendTimeout { + // Do not feed an already-stale backlog into a briefly writable + // QUIC queue between watchdog ticks. Retire this connection. + s.closed = true + s.mu.Unlock() + s.b.peerQueueDiscarded.Add(1) + s.timeout() + return + } + s.mu.Unlock() + err := s.conn.SendDatagram(job.payload) + s.mu.Lock() + s.sendingSince = time.Time{} + s.mu.Unlock() + if err != nil { + s.b.recordSendError(s.peer, err) + var tooLarge *quic.DatagramTooLargeError + if errors.As(err, &tooLarge) { + continue + } + s.b.dropConnection(s.peer.Identity, s.conn) + s.b.queueConnect(s.peer.Identity) + return + } + s.b.sends.Add(1) + } + } +} + +func (s *votorPeerSender) stats() VotorPeerQueueStats { + s.mu.Lock() + defer s.mu.Unlock() + var sendingFor time.Duration + if !s.sendingSince.IsZero() { + sendingFor = time.Since(s.sendingSince) + } + return VotorPeerQueueStats{ + Identity: s.peer.Identity, Address: s.peer.Addr.String(), + Queued: len(s.queue), SendingFor: sendingFor, + LastQueueDelay: s.lastQueueDelay, MaxQueueDelay: s.maxQueueDelay, QueueDrops: s.queueDrops, + } +} + +func (b *VotorBroadcaster) expirePeerSends(now time.Time) { + senders, _ := b.connectedSenders() + for _, s := range senders { + s.mu.Lock() + expired := !s.closed && !s.sendingSince.IsZero() && s.lastQueueDelay+now.Sub(s.sendingSince) >= votorSendTimeout + if expired { + // Serialize with send completion so a late watchdog cannot close a + // later, unrelated send after the blocked operation has finished. + s.closed = true + } + s.mu.Unlock() + if expired { + s.timeout() + } + } +} + +// Caller must first claim the timeout by setting closed under s.mu. This makes +// the dequeue check and watchdog mutually exclusive and counts one timeout. +func (s *votorPeerSender) timeout() { + s.b.peerSendTimeouts.Add(1) + // Closing wakes SendDatagram without leaking a timeout goroutine. + s.b.dropConnection(s.peer.Identity, s.conn) + s.b.queueConnect(s.peer.Identity) +} diff --git a/pkg/alpenglow/peer_sender_recovery_test.go b/pkg/alpenglow/peer_sender_recovery_test.go new file mode 100644 index 000000000..3f88446ea --- /dev/null +++ b/pkg/alpenglow/peer_sender_recovery_test.go @@ -0,0 +1,136 @@ +package alpenglow + +import ( + "context" + "crypto/ed25519" + "crypto/tls" + "net" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Real authenticated connection with no reconciliation loop or connection +// workers. Any reconnect job must come from the sender, never the periodic tick. +func passiveVotorSender(t *testing.T) (*VotorBroadcaster, *votorPeerSender, *Receiver) { + t.Helper() + serverIdentity := ed25519.NewKeyFromSeed(bytesOf(181, ed25519.SeedSize)) + r, err := NewReceiver(ReceiverConfig{ + BindAddr: "127.0.0.1:0", Identity: serverIdentity, LogInterval: -1, + AdmitPeer: func(solana.PublicKey) bool { return true }, + }, NewObserver()) + require.NoError(t, err) + runVotorReceiver(t, r) + cert, err := newVotorQUICCertificate(ed25519.NewKeyFromSeed(bytesOf(182, ed25519.SeedSize))) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + peer := VotorPeer{Identity: testVotorPubkey(serverIdentity), Addr: r.Addr().(*net.UDPAddr)} + b := &VotorBroadcaster{ + ctx: ctx, cancel: cancel, done: make(chan struct{}), + tlsConfig: &tls.Config{Certificates: []tls.Certificate{cert}, NextProtos: []string{VotorQUICALPN}, MinVersion: tls.VersionTLS13, InsecureSkipVerify: true}, + quicConfig: newVotorQUICConfig(), + jobs: make(chan votorPeerJob, 2), desired: map[solana.PublicKey]VotorPeer{peer.Identity: peer}, + conns: make(map[solana.PublicKey]votorConnection), dialing: make(map[solana.PublicKey]*votorDial), connectQueued: make(map[solana.PublicKey]struct{}), + } + t.Cleanup(func() { require.NoError(t, b.Close()) }) + _, err = b.connection(peer) + require.NoError(t, err) + b.connMu.Lock() + sender := b.conns[peer.Identity].sender + b.connMu.Unlock() + return b, sender, r +} + +func TestVotorPeerReconnectOnRemoteCloseWithoutReconcile(t *testing.T) { + for _, duringDequeue := range []bool{false, true} { + name := "idle" + if duringDequeue { + name = "dequeue" + } + t.Run(name, func(t *testing.T) { + b, s, receiver := passiveVotorSender(t) + if duringDequeue { + func() { + s.mu.Lock() + defer s.mu.Unlock() + s.queue <- votorDatagram{payload: []byte{1}, queuedAt: time.Now()} + // Force the job branch to win select, then close remotely + // while the sender is waiting to check the connection state. + require.Eventually(t, func() bool { return len(s.queue) == 0 }, time.Second, time.Millisecond) + require.NoError(t, receiver.Close()) + select { + case <-s.conn.Context().Done(): + case <-time.After(time.Second): + t.Fatal("remote close not observed") + } + }() + } else { + require.NoError(t, receiver.Close()) + } + select { + case <-s.done: + case <-time.After(time.Second): + t.Fatal("sender did not exit") + } + select { + case job := <-b.jobs: + require.Equal(t, s.peer.Identity, job.peer.Identity) + default: + t.Fatal("sender exited without requesting reconnect") + } + require.Empty(t, b.jobs, "only one reconnect request per peer") + }) + } +} + +func TestVotorPeerDeadlineIncludesQueueAge(t *testing.T) { + for _, tc := range []struct { + name string + queued, sending time.Duration + active, expired bool + }{ + {"progress_does_not_reset_age", 950 * time.Millisecond, 100 * time.Millisecond, true, true}, + {"below_deadline", 800 * time.Millisecond, 100 * time.Millisecond, true, false}, + {"blocked_call", 0, time.Second, true, true}, + {"completed_send", 2 * time.Second, 0, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + b, s, _ := passiveVotorSender(t) + now := time.Now() + s.mu.Lock() + if tc.active { + s.sendingSince = now.Add(-tc.sending) + } + s.lastQueueDelay = tc.queued + s.mu.Unlock() + // Inject a deterministic clock/state boundary. There is no actual + // datagram in progress and no timer goroutine in this fixture. + b.expirePeerSends(now) + require.Equal(t, tc.expired, s.conn.Context().Err() != nil) + if tc.expired { + require.EqualValues(t, 1, b.peerSendTimeouts.Load()) + b.expirePeerSends(now.Add(time.Second)) + require.EqualValues(t, 1, b.peerSendTimeouts.Load()) + } else { + require.Zero(t, b.peerSendTimeouts.Load()) + } + }) + } +} + +func TestVotorPeerRejectsAgedQueueBeforeQUICEnqueue(t *testing.T) { + b, s, _ := passiveVotorSender(t) + s.enqueue(votorDatagram{payload: []byte{1}, queuedAt: time.Now().Add(-2 * time.Second)}) + select { + case <-s.done: + case <-time.After(time.Second): + t.Fatal("aged queue did not retire its connection") + } + require.Error(t, s.conn.Context().Err()) + require.Zero(t, b.sends.Load(), "stale backlog must not enter the QUIC queue") + require.EqualValues(t, 1, b.peerSendTimeouts.Load()) + require.EqualValues(t, 1, b.peerQueueDiscarded.Load()) + require.Len(t, b.jobs, 1) +} diff --git a/pkg/alpenglow/peer_sender_test.go b/pkg/alpenglow/peer_sender_test.go new file mode 100644 index 000000000..dda145c4a --- /dev/null +++ b/pkg/alpenglow/peer_sender_test.go @@ -0,0 +1,245 @@ +package alpenglow + +import ( + "crypto/ed25519" + "net" + "runtime" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Real QUIC traffic: one path is blackholed after authentication while the +// other stays healthy. This reproduces the original shared-worker failure. +func TestVotorBroadcasterIsolatesBlockedPeer(t *testing.T) { + for _, action := range []string{"reconnect", "close", "depart", "move"} { + t.Run(action, func(t *testing.T) { testVotorBlockedPeer(t, action) }) + } +} + +func testVotorBlockedPeer(t *testing.T, action string) { + const markerSlot = 999999 + marker := make(chan time.Time, 1) + badMarker := make(chan time.Time, 1) + makeReceiver := func(seed byte, record bool) (*Receiver, solana.PublicKey) { + identity := ed25519.NewKeyFromSeed(bytesOf(seed, ed25519.SeedSize)) + r, err := NewReceiver(ReceiverConfig{ + BindAddr: "127.0.0.1:0", Identity: identity, LogInterval: -1, + MaxDatagramsPerSecond: 100000, + AdmitPeer: func(solana.PublicKey) bool { return true }, + AdmitMessage: func(_ solana.PublicKey, m Message) (Message, bool) { + if m.Slot() == markerSlot { + target := badMarker + if record { + target = marker + } + select { + case target <- time.Now(): + default: + } + } + return Message{}, false + }, + }, NewObserver()) + require.NoError(t, err) + runVotorReceiver(t, r) + return r, testVotorPubkey(identity) + } + badReceiver, badID := makeReceiver(191, false) + goodReceiver, goodID := makeReceiver(192, true) + badAddr := badReceiver.Addr().(*net.UDPAddr) + proxy, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + var blackhole atomic.Bool + proxyDone := make(chan struct{}) + go func() { + defer close(proxyDone) + buf := make([]byte, 65536) + var client *net.UDPAddr + for { + n, from, err := proxy.ReadFromUDP(buf) + if err != nil { + return + } + if blackhole.Load() { + continue + } + if from.String() == badAddr.String() { + if client != nil { + _, _ = proxy.WriteToUDP(buf[:n], client) + } + } else { + client = from + _, _ = proxy.WriteToUDP(buf[:n], badAddr) + } + } + }() + t.Cleanup(func() { _ = proxy.Close(); <-proxyDone }) + badPeer := VotorPeer{Identity: badID, Addr: proxy.LocalAddr().(*net.UDPAddr)} + goodPeer := VotorPeer{Identity: goodID, Addr: goodReceiver.Addr().(*net.UDPAddr)} + peers := newMutableVotorPeers([]VotorPeer{badPeer, goodPeer}) + b, err := NewVotorBroadcaster(VotorBroadcasterConfig{ + Identity: ed25519.NewKeyFromSeed(bytesOf(193, ed25519.SeedSize)), + Peers: peers.Snapshot, + Workers: defaultVotorConnectWorkers, + }) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, b.Close()) }) + require.Eventually(t, func() bool { return b.Stats().Connections == 2 }, 3*time.Second, 5*time.Millisecond) + badConn, ok := b.establishedConnection(badPeer) + require.True(t, ok) + // Establish an actual healthy delivery before introducing the fault. + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x41), 3))) + select { + case <-marker: + case <-time.After(time.Second): + t.Fatal("healthy baseline failed") + } + select { + case <-badMarker: + case <-time.After(time.Second): + t.Fatal("proxied baseline failed") + } + blackholedAt := time.Now() + blackhole.Store(true) + for slot := uint64(1); slot <= 256; slot++ { + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(slot), testSignatureSeq(0x41), 3))) + } + blockedWorkers := func() int { + buf := make([]byte, 2<<20) + stack := string(buf[:runtime.Stack(buf, true)]) + count := 0 + for _, goroutine := range strings.Split(stack, "\n\n") { + if strings.Contains(goroutine, "(*datagramQueue).Add") && strings.Contains(goroutine, "(*votorPeerSender).run") { + count++ + } + } + return count + } + require.Eventually(t, func() bool { return blockedWorkers() == 1 }, 3*time.Second, 5*time.Millisecond) + before := b.Stats() + require.Zero(t, before.MessagesDropped) + goodConn, ok := b.establishedConnection(goodPeer) + require.True(t, ok) + b.connMu.Lock() + badSender := b.conns[badID].sender + b.connMu.Unlock() + start := time.Now() + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x42), 3))) + select { + case received := <-marker: + t.Logf("Healthy marker received in %s while other peer's SendDatagram is blocked", received.Sub(start)) + case <-time.After(250 * time.Millisecond): + t.Fatal("stalled peer delayed healthy delivery") + } + require.NoError(t, badConn.Context().Err(), "marker must arrive before watchdog releases stalled peer") + if action != "reconnect" { + switch action { + case "close": + closed := make(chan struct{}) + go func() { _ = b.Close(); close(closed) }() + select { + case <-closed: + case <-time.After(500 * time.Millisecond): + t.Fatal("Close waited for stalled SendDatagram") + } + case "depart": + peers.Set([]VotorPeer{goodPeer}) + b.reconcilePeers() + case "move": + replacement, _ := makeReceiver(191, false) + badPeer.Addr = replacement.Addr().(*net.UDPAddr) + peers.Set([]VotorPeer{badPeer, goodPeer}) + b.reconcilePeers() + } + select { + case <-badSender.done: + case <-time.After(500 * time.Millisecond): + t.Fatal("old sender did not stop") + } + require.Error(t, badConn.Context().Err()) + require.Empty(t, badSender.queue) + require.Positive(t, b.Stats().PeerQueueDiscarded) + if action == "close" { + return + } + if action == "move" { + require.Eventually(t, func() bool { + conn, ok := b.establishedConnection(badPeer) + return ok && conn != badConn + }, 3*time.Second, 5*time.Millisecond) + } + currentGood, ok := b.establishedConnection(goodPeer) + require.True(t, ok) + require.Same(t, goodConn, currentGood) + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x44), 3))) + select { + case <-marker: + case <-time.After(time.Second): + t.Fatal("healthy delivery stopped after peer change") + } + if action == "move" { + select { + case <-badMarker: + case <-time.After(time.Second): + t.Fatal("replacement address did not receive new vote") + } + } + return + } + // Deliberately fill just the stalled peer's bounded queue. Healthy peers + // remain independent even when the failed peer's copies are rejected. + payload, err := EncodeMessage(NewVoteMessage(NewSkipVote(123), testSignatureSeq(0x42), 3)) + require.NoError(t, err) + for range 2 * defaultVotorPeerSendQueue { + badSender.enqueue(votorDatagram{payload: payload, queuedAt: time.Now()}) + } + require.Equal(t, defaultVotorPeerSendQueue, len(badSender.queue)) + require.Positive(t, b.Stats().PeerQueueDrops) + require.Equal(t, b.Stats().PeerQueueDrops, b.Stats().MessagesDropped) + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x43), 3))) + select { + case <-marker: + case <-time.After(250 * time.Millisecond): + t.Fatal("full peer queue delayed healthy delivery") + } + // Queue age is measured from fanout, so PTO progress no longer restarts + // the one-second budget. Allow two seconds of test scheduling slack beyond + // one watchdog tick, measured from the fault rather than this assertion. + remaining := time.Until(blackholedAt.Add(votorSendTimeout + votorSendWatchInterval + 2*time.Second)) + require.Positive(t, remaining) + require.Eventually(t, func() bool { return badConn.Context().Err() != nil }, remaining, 5*time.Millisecond) + t.Logf("Blackholed peer retired after %s", time.Since(blackholedAt)) + select { + case <-badSender.done: + case <-time.After(time.Second): + t.Fatal("stalled sender leaked after watchdog closed its connection") + } + require.EqualValues(t, 1, b.Stats().PeerSendTimeouts) + require.Positive(t, b.Stats().PeerQueueDiscarded) + require.Empty(t, badSender.queue) + // The old queue must never be resurrected on a replacement connection. + blackhole.Store(false) + require.Eventually(t, func() bool { + conn, ok := b.establishedConnection(badPeer) + return ok && conn != badConn + }, 5*time.Second, 10*time.Millisecond) + currentGood, ok := b.establishedConnection(goodPeer) + require.True(t, ok) + require.Same(t, goodConn, currentGood) + require.NoError(t, b.Enqueue(NewVoteMessage(NewSkipVote(markerSlot), testSignatureSeq(0x44), 3))) + select { + case <-marker: + case <-time.After(time.Second): + t.Fatal("healthy peer did not continue after reconnect") + } + select { + case <-badMarker: + case <-time.After(time.Second): + t.Fatal("reconnected peer did not receive new vote") + } +} diff --git a/pkg/consensus/voter.go b/pkg/consensus/voter.go index b77a7689b..e5a220d96 100644 --- a/pkg/consensus/voter.go +++ b/pkg/consensus/voter.go @@ -53,28 +53,33 @@ type VotingConfig struct { // NetworkLandedVotes counts unique persisted votes whose rank appeared in the // exact BLS-verified certificate proof received over Votor QUIC. type VotingStats struct { - Enabled bool `json:"enabled"` - VotesCastThisRun uint64 `json:"votes_cast_this_run"` - NetworkLandedVotes uint64 `json:"network_landed_votes"` - LastNetworkLandedSlot uint64 `json:"last_network_landed_slot,omitempty"` - LastNetworkLandedVoteType alpenglow.VoteType `json:"last_network_landed_vote_type,omitempty"` - LastNetworkCertificateType alpenglow.CertificateType `json:"last_network_certificate_type,omitempty"` - LastNetworkLandedAt time.Time `json:"last_network_landed_at,omitempty"` - BroadcastMessagesQueued uint64 `json:"broadcast_messages_queued"` - BroadcastMessagesDropped uint64 `json:"broadcast_messages_dropped"` - BroadcastPeerSends uint64 `json:"broadcast_peer_sends"` - BroadcastPeerSendsSkipped uint64 `json:"broadcast_peer_sends_skipped"` - BroadcastPeerSendErrors uint64 `json:"broadcast_peer_send_errors"` - BroadcastDesiredPeers int `json:"broadcast_desired_peers"` - BroadcastActiveConnections int `json:"broadcast_active_connections"` - BroadcastPendingConnections int `json:"broadcast_pending_connections"` - BroadcastConnectionAttempts uint64 `json:"broadcast_connection_attempts"` - BroadcastConnectionErrors uint64 `json:"broadcast_connection_errors"` - BroadcastConnectionJobsDropped uint64 `json:"broadcast_connection_jobs_dropped"` - BroadcastLastPeerSendError string `json:"broadcast_last_peer_send_error,omitempty"` - BroadcastLastPeerSendErrorAt time.Time `json:"broadcast_last_peer_send_error_at,omitempty"` - BroadcastLastConnectionError string `json:"broadcast_last_connection_error,omitempty"` - BroadcastLastConnectionErrorAt time.Time `json:"broadcast_last_connection_error_at,omitempty"` + Enabled bool `json:"enabled"` + VotesCastThisRun uint64 `json:"votes_cast_this_run"` + NetworkLandedVotes uint64 `json:"network_landed_votes"` + LastNetworkLandedSlot uint64 `json:"last_network_landed_slot,omitempty"` + LastNetworkLandedVoteType alpenglow.VoteType `json:"last_network_landed_vote_type,omitempty"` + LastNetworkCertificateType alpenglow.CertificateType `json:"last_network_certificate_type,omitempty"` + LastNetworkLandedAt time.Time `json:"last_network_landed_at,omitempty"` + BroadcastMessagesQueued uint64 `json:"broadcast_messages_queued"` + BroadcastMessagesDropped uint64 `json:"broadcast_messages_dropped"` + BroadcastPeerSends uint64 `json:"broadcast_peer_sends"` + BroadcastPeerSendsSkipped uint64 `json:"broadcast_peer_sends_skipped"` + BroadcastPeerSendErrors uint64 `json:"broadcast_peer_send_errors"` + BroadcastPeerQueueDrops uint64 `json:"broadcast_peer_queue_drops"` + BroadcastPeerQueueDiscarded uint64 `json:"broadcast_peer_queue_discarded"` + BroadcastPeerSendTimeouts uint64 `json:"broadcast_peer_send_timeouts"` + BroadcastPeerQueueMaxDelay time.Duration `json:"broadcast_peer_queue_max_delay"` + BroadcastPeerQueues []alpenglow.VotorPeerQueueStats `json:"broadcast_peer_queues,omitempty"` + BroadcastDesiredPeers int `json:"broadcast_desired_peers"` + BroadcastActiveConnections int `json:"broadcast_active_connections"` + BroadcastPendingConnections int `json:"broadcast_pending_connections"` + BroadcastConnectionAttempts uint64 `json:"broadcast_connection_attempts"` + BroadcastConnectionErrors uint64 `json:"broadcast_connection_errors"` + BroadcastConnectionJobsDropped uint64 `json:"broadcast_connection_jobs_dropped"` + BroadcastLastPeerSendError string `json:"broadcast_last_peer_send_error,omitempty"` + BroadcastLastPeerSendErrorAt time.Time `json:"broadcast_last_peer_send_error_at,omitempty"` + BroadcastLastConnectionError string `json:"broadcast_last_connection_error,omitempty"` + BroadcastLastConnectionErrorAt time.Time `json:"broadcast_last_connection_error_at,omitempty"` } type voterEventKind uint8 @@ -1208,6 +1213,11 @@ func (v *alpenglowVoter) snapshot() VotingStats { stats.BroadcastPeerSends = broadcast.PeerSends stats.BroadcastPeerSendsSkipped = broadcast.PeerSendsSkipped stats.BroadcastPeerSendErrors = broadcast.PeerSendErrors + stats.BroadcastPeerQueueDrops = broadcast.PeerQueueDrops + stats.BroadcastPeerQueueDiscarded = broadcast.PeerQueueDiscarded + stats.BroadcastPeerSendTimeouts = broadcast.PeerSendTimeouts + stats.BroadcastPeerQueueMaxDelay = broadcast.PeerQueueMaxDelay + stats.BroadcastPeerQueues = broadcast.PeerQueues stats.BroadcastDesiredPeers = broadcast.DesiredPeers stats.BroadcastActiveConnections = broadcast.Connections stats.BroadcastPendingConnections = broadcast.PendingConnections From 318ddb22f2d34cd551c3dbb2e99e2b615b3bbb7d Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Thu, 24 Sep 2026 00:16:49 -0500 Subject: [PATCH 13/13] alpenglow: separate opt-in signing reservations and ordered history persistence Preserve reviewed persistence changes from fde37516, 75f4b7e7, c2cbc4a8 and e10b9537; transport lives in the certificate/delivery prerequisite. --- .github/workflows/go_build.yml | 28 ++ cmd/mithril/node/node.go | 103 +++--- cmd/mithril/node/vote_startup.go | 40 +++ cmd/mithril/node/vote_startup_test.go | 73 ++++ config.example.toml | 10 + docs/reserved-vote-history.md | 162 +++++++++ docs/vote-delivery-persistence-evidence.md | 14 + pkg/alpenglow/vote_history.go | 113 ++++-- pkg/alpenglow/vote_history_snapshot_test.go | 49 +++ pkg/alpenglow/vote_reservation.go | 104 ++++++ pkg/alpenglow/vote_reservation_test.go | 71 ++++ pkg/blockprod/leader.go | 6 + .../leader_signing_reservation_test.go | 17 + pkg/config/config.go | 2 + pkg/consensus/engine.go | 40 ++- pkg/consensus/vote_history_writer.go | 125 +++++++ pkg/consensus/vote_history_writer_test.go | 201 +++++++++++ pkg/consensus/vote_reservation.go | 296 ++++++++++++++++ pkg/consensus/vote_reservation_test.go | 330 ++++++++++++++++++ pkg/consensus/voter.go | 279 +++++++++++---- pkg/consensus/voter_finality_ordering_test.go | 192 ++++++++++ pkg/consensus/voter_wait_slot_test.go | 91 +++++ 22 files changed, 2209 insertions(+), 137 deletions(-) create mode 100644 cmd/mithril/node/vote_startup.go create mode 100644 cmd/mithril/node/vote_startup_test.go create mode 100644 docs/reserved-vote-history.md create mode 100644 docs/vote-delivery-persistence-evidence.md create mode 100644 pkg/alpenglow/vote_history_snapshot_test.go create mode 100644 pkg/alpenglow/vote_reservation.go create mode 100644 pkg/alpenglow/vote_reservation_test.go create mode 100644 pkg/blockprod/leader_signing_reservation_test.go create mode 100644 pkg/consensus/vote_history_writer.go create mode 100644 pkg/consensus/vote_history_writer_test.go create mode 100644 pkg/consensus/vote_reservation.go create mode 100644 pkg/consensus/vote_reservation_test.go create mode 100644 pkg/consensus/voter_finality_ordering_test.go create mode 100644 pkg/consensus/voter_wait_slot_test.go diff --git a/.github/workflows/go_build.yml b/.github/workflows/go_build.yml index 50383467f..4fa7306c9 100644 --- a/.github/workflows/go_build.yml +++ b/.github/workflows/go_build.yml @@ -17,3 +17,31 @@ jobs: - name: Build run: go build -v ./cmd/mithril + + regression-tests: + runs-on: ubuntu-latest + timeout-minutes: 20 + permissions: + contents: read + env: + GOMAXPROCS: "2" + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: 1.26.4 + + - name: Voting, checkpoint, streaming and scheduler race regressions + # Run the complete affected package suites, including subprocess crash + # recovery and cancellation tests. The independent sealevel suite has + # known base-branch failures documented in the validation report. + run: >- + go test -race -p 2 -count=1 + ./pkg/alpenglow ./pkg/consensus ./pkg/replay + ./pkg/turbine ./pkg/sigverify ./pkg/blockprod/... + ./cmd/mithril/node ./cmd/mithril/configcmd + + - name: Vote-program deque ownership race regression + run: go test -race -count=1 ./pkg/sealevel -run '^TestProcessNewVoteStateOwnsRetainedDeque$' diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index c452d494e..1f6a21d8e 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -103,6 +103,9 @@ var ( validatorTPUQUICBind string validatorAdvertisedIP string validatorSigverifyWorkers int + validatorWaitToVoteSlot uint64 + validatorReservedHistory bool + validatorInitializeReservation bool // Mode thresholds blockNearTipThreshold int // Enter near-tip when gap <= this @@ -547,6 +550,9 @@ func init() { Run.Flags().StringVar(&validatorTPUQUICBind, "tpu-quic-bind-addr", "", "Validator TPU QUIC listen address (default 0.0.0.0:8004)") Run.Flags().StringVar(&validatorAdvertisedIP, "validator-advertised-ip", "", "Public IP advertised for validator TPU QUIC") Run.Flags().IntVar(&validatorSigverifyWorkers, "tpu-sigverify-workers", 0, "TPU signature verification workers (0 = GOMAXPROCS)") + Run.Flags().BoolVar(&validatorReservedHistory, "reserved-vote-history", false, "Use durable signing reservations with unsynchronized per-vote history writes") + Run.Flags().BoolVar(&validatorInitializeReservation, "initialize-vote-reservation", false, "Enroll complete synchronous vote history in reserved mode (one-time migration)") + Run.Flags().Uint64Var(&validatorWaitToVoteSlot, "wait-to-vote-slot", 0, "Do not cast new votes below this slot; the automatic startup cutoff still applies (0 = automatic only)") // [tuning] section flags Run.Flags().Uint64Var(¶mArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)") @@ -639,6 +645,11 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { if err := config.InitConfig(); err != nil { return err } + if slot, err := configuredWaitToVoteSlot(cmd); err != nil { + return err + } else { + validatorWaitToVoteSlot = slot + } // Check if a CLI flag was explicitly set by the user flagChanged := func(name string) bool { @@ -2642,52 +2653,9 @@ postBootstrap: } global.SeedWallClockSlot(wallClockSeed) startupWallSlot := global.WallClockSlot() - waitToVoteSlot := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots - if waitToVoteSlot <= math.MaxUint64-2*alpenglow.LeaderWindowSlots { - waitToVoteSlot += 2 * alpenglow.LeaderWindowSlots - } else { - waitToVoteSlot = math.MaxUint64 - } - mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d wait_to_vote=%d", startupWallSlot, waitToVoteSlot) + waitToVoteSlot := effectiveWaitToVoteSlot(startupWallSlot, validatorWaitToVoteSlot) + mlog.Log.Infof("ALPENGLOW voting startup watermark: wall_clock=%d configured_wait_to_vote=%d wait_to_vote=%d", startupWallSlot, validatorWaitToVoteSlot, waitToVoteSlot) - identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey() - if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{ - Identity: validatorIdentity, - AuthorizedVoter: validatorAuthorizedVoter, - VoteAccount: validatorVoteAccount, - HistoryDir: blockstorePath, - EpochForSlot: epochSchedule.GetEpoch, - SlotDuration: blockprod.AlpenglowSlotDuration, - WaitToVoteSlot: waitToVoteSlot, - ReadyToVote: func(slot uint64) bool { - wallSlot := global.WallClockSlot() - if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok { - wallSlot = liveSlot - } - return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots - }, - Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer { - peers := make([]alpenglow.VotorPeer, 0, len(validators)) - seen := make(map[solana.PublicKey]struct{}, len(validators)) - for _, validator := range validators { - if validator.Stake == 0 || validator.NodePubkey == identityPubkey { - continue - } - addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey) - if !ok { - continue - } - if _, duplicate := seen[validator.NodePubkey]; duplicate { - continue - } - seen[validator.NodePubkey] = struct{}{} - peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr}) - } - return peers - }, - }); err != nil { - klog.Fatalf("enable Alpenglow voting: %v", err) - } broadcaster, err := turbine.NewTurbineBroadcaster(turbine.TurbineBroadcasterConfig{ Self: solana.PrivateKey(validatorIdentity).PublicKey(), Peers: sharedGossip, @@ -2744,6 +2712,50 @@ postBootstrap: mlog.Log.Warnf("validator gossip TPU advertisement: %v", err) } + // Bind and validate local transports before consuming the durable clean + // voting marker. Startup configuration failures must not force recovery. + identityPubkey := solana.PrivateKey(validatorIdentity).PublicKey() + if err := consensusEngine.EnableVoting(consensusengine.VotingConfig{ + Identity: validatorIdentity, + AuthorizedVoter: validatorAuthorizedVoter, + VoteAccount: validatorVoteAccount, + HistoryDir: blockstorePath, + ReservedHistory: validatorReservedHistory, + InitializeVoteReservation: validatorInitializeReservation, + Genesis: solana.MustHashFromBase58(networkGenesisHash), + EpochForSlot: epochSchedule.GetEpoch, + SlotDuration: blockprod.AlpenglowSlotDuration, + WaitToVoteSlot: waitToVoteSlot, + ReadyToVote: func(slot uint64) bool { + wallSlot := global.WallClockSlot() + if liveSlot, ok := consensusEngine.AlpenglowLiveSlot(); ok { + wallSlot = liveSlot + } + return slot >= wallSlot || wallSlot-slot <= alpenglow.LeaderWindowSlots + }, + Peers: func(validators []alpenglow.ValidatorStake) []alpenglow.VotorPeer { + peers := make([]alpenglow.VotorPeer, 0, len(validators)) + seen := make(map[solana.PublicKey]struct{}, len(validators)) + for _, validator := range validators { + if validator.Stake == 0 || validator.NodePubkey == identityPubkey { + continue + } + addr, ok := sharedGossip.LookupAlpenglow(validator.NodePubkey) + if !ok { + continue + } + if _, duplicate := seen[validator.NodePubkey]; duplicate { + continue + } + seen[validator.NodePubkey] = struct{}{} + peers = append(peers, alpenglow.VotorPeer{Identity: validator.NodePubkey, Addr: addr}) + } + return peers + }, + }); err != nil { + klog.Fatalf("enable Alpenglow voting: %v", err) + } + rewardBuilder := rewardcerts.NewBuilder(rewardcerts.BuilderConfig{ RootSlot: global.Slot, BeforeBuild: consensusEngine.FlushAlpenglowRewardVotes, @@ -2796,6 +2808,7 @@ postBootstrap: } }, ProductionParent: consensusEngine.AlpenglowBlockProductionParent, + CanSignSlot: consensusEngine.AlpenglowCanSignLeaderSlot, CurrentSlot: func() uint64 { if slot, ok := consensusEngine.AlpenglowLiveSlot(); ok { return slot diff --git a/cmd/mithril/node/vote_startup.go b/cmd/mithril/node/vote_startup.go new file mode 100644 index 000000000..547e55194 --- /dev/null +++ b/cmd/mithril/node/vote_startup.go @@ -0,0 +1,40 @@ +package node + +import ( + "fmt" + "math" + "strconv" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/spf13/cobra" +) + +func configuredWaitToVoteSlot(cmd *cobra.Command) (uint64, error) { + if flag := cmd.Flags().Lookup("wait-to-vote-slot"); flag != nil && flag.Changed { + return cmd.Flags().GetUint64("wait-to-vote-slot") + } + const key = "validator.wait_to_vote_slot" + if !config.IsSet(key) { + return 0, nil + } + // Unlike GetUint64, parsing explicitly must not turn an invalid operator + // cutoff into zero and silently remove the requested voting restriction. + slot, err := strconv.ParseUint(config.GetString(key), 10, 64) + if err != nil { + return 0, fmt.Errorf("%s must be an unsigned 64-bit slot: %w", key, err) + } + return slot, nil +} + +// The operator cutoff can postpone voting but cannot weaken the existing +// startup guard. Equality permits voting, subject to all other Votor checks. +func effectiveWaitToVoteSlot(startupWallSlot, configured uint64) uint64 { + automatic := startupWallSlot - startupWallSlot%alpenglow.LeaderWindowSlots + if automatic <= math.MaxUint64-2*alpenglow.LeaderWindowSlots { + automatic += 2 * alpenglow.LeaderWindowSlots + } else { + automatic = math.MaxUint64 + } + return max(automatic, configured) +} diff --git a/cmd/mithril/node/vote_startup_test.go b/cmd/mithril/node/vote_startup_test.go new file mode 100644 index 000000000..d86a02528 --- /dev/null +++ b/cmd/mithril/node/vote_startup_test.go @@ -0,0 +1,73 @@ +package node + +import ( + "math" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestConfiguredWaitToVoteSlot(t *testing.T) { + for _, tc := range []struct { + name, toml, cli string + want uint64 + invalid bool + }{ + {name: "default"}, + {name: "toml", toml: "wait_to_vote_slot = 1234", want: 1234}, + {name: "cli wins", toml: "wait_to_vote_slot = 1234", cli: "5678", want: 5678}, + {name: "explicit zero wins", toml: "wait_to_vote_slot = 1234", cli: "0"}, + {name: "maximum CLI", cli: "18446744073709551615", want: math.MaxUint64}, + {name: "negative TOML", toml: "wait_to_vote_slot = -1", invalid: true}, + {name: "fractional TOML", toml: "wait_to_vote_slot = 1.5", invalid: true}, + {name: "malformed TOML value", toml: `wait_to_vote_slot = "oops"`, invalid: true}, + {name: "empty TOML value", toml: `wait_to_vote_slot = ""`, invalid: true}, + {name: "overflow TOML value", toml: `wait_to_vote_slot = "18446744073709551616"`, invalid: true}, + {name: "negative CLI", cli: "-1", invalid: true}, + {name: "overflow CLI", cli: "18446744073709551616", invalid: true}, + } { + t.Run(tc.name, func(t *testing.T) { + viper.Reset() + t.Cleanup(viper.Reset) + config.ApplyDefaults(viper.GetViper()) + viper.SetConfigType("toml") + require.NoError(t, viper.ReadConfig(strings.NewReader("[validator]\n"+tc.toml))) + cmd := &cobra.Command{} + cmd.Flags().Uint64("wait-to-vote-slot", 0, "") + var err error + if tc.cli != "" { + err = cmd.Flags().Set("wait-to-vote-slot", tc.cli) + } + var got uint64 + if err == nil { + got, err = configuredWaitToVoteSlot(cmd) + } + if tc.invalid { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tc.want, got) + }) + } + require.NotNil(t, Run.Flags().Lookup("wait-to-vote-slot")) +} + +func TestEffectiveWaitToVoteSlot(t *testing.T) { + for _, tc := range []struct{ startup, configured, want uint64 }{ + {100, 0, 108}, + {103, 0, 108}, + {103, 104, 108}, + {103, 108, 108}, + {103, 123, 123}, // Operator cutoff need not align with a leader window. + {103, math.MaxUint64, math.MaxUint64}, + {math.MaxUint64 - 7, 0, math.MaxUint64}, + {math.MaxUint64, 0, math.MaxUint64}, + } { + require.Equal(t, tc.want, effectiveWaitToVoteSlot(tc.startup, tc.configured), "%+v", tc) + } +} diff --git a/config.example.toml b/config.example.toml index 7e3833087..f46e78549 100644 --- a/config.example.toml +++ b/config.example.toml @@ -328,6 +328,16 @@ name = "mithril" # Signature-verification workers (0 = GOMAXPROCS). tpu_sigverify_workers = 0 + # Optional minimum slot for NEW votes (inclusive), also --wait-to-vote-slot. + # Useful when rejoining after recovery. CLI overrides this setting. + # Zero adds no operator cutoff; the automatic startup cutoff and normal + # consensus checks still apply. A lower value cannot bypass those checks. + # Replay/repair continue while waiting. Previously recorded, authenticated + # votes can still be restored/rebroadcast under the existing recovery rules. + # This does not coordinate a cluster restart or wait for supermajority, + # and does not allow resetting a corrupt vote-history file. + wait_to_vote_slot = 0 + # ============================================================================ # [consensus] - Alpenglow Consensus # ============================================================================ diff --git a/docs/reserved-vote-history.md b/docs/reserved-vote-history.md new file mode 100644 index 000000000..bd94e2e57 --- /dev/null +++ b/docs/reserved-vote-history.md @@ -0,0 +1,162 @@ +# Voting persistence and crash recovery + +## Intended guarantee + +A crash must not lead to conflicting externally published votes or reserved-mode +leader actions because the validator forgot its earlier local decisions. The design permits +loss of recent detailed history and sacrifices voting availability when its +completeness is uncertain. It does not promise immediate restart voting, that +every signed vote reaches disk, or recovery from rolled-back safety files. +Normal anti-equivocation, execution, parent and validator-binding checks remain +necessary; this is a persistence contract, not a proof of the entire protocol. + +The failure model includes process termination and host/power failure, provided +successful file and directory syncs survive, the current safety files are +preserved, and only one fenced owner uses the signing identity. Software tests +exercise the recovery decisions; they do not qualify actual storage against +power loss. Valid signatures on saved files establish integrity, not freshness. + +## Two different publication guarantees + +| Mode | Required before a vote can escape to the pool/network | What a restart may trust | +| --- | --- | --- | +| Default synchronous history | Exact validated history is written, file-synced, renamed and directory-synced before local pool admission or network enqueue. | Retained exact decisions and their rooted boundary, subject to normal restoration checks. | +| Opt-in reserved history | The vote's slot is covered by an acknowledged durable reservation before signing. Its exact history snapshot is prepared and queued before publication, without waiting for per-vote I/O. | The startup reservation bound, unless a separately validated clean-history seal proves exact retained history. | + +In synchronous mode, BLS bytes may be computed privately in RAM before history +is synced. The guarantee is **persist before publication**, including local +pool admission because it can publish a certificate. In reserved mode, a +successful queue submission, background rename, or `written` counter is **not** +a durable acknowledgement of that vote. Replacing a whole file atomically is +not the same as making its bytes/directory entry survive power loss. + +Both modes retain complete decisions in memory while running and prune them +only through the normal verified-root rules. The synchronous history guarantee +covers votes; it is not a complete journal of produced leader blocks. The +additional leader reservation barrier applies only in reserved mode. + +## Reserved-mode restart rule + +Let **H** be the reservation's `Through` value loaded at startup, **F** the +verified finality/checkpoint floor, and **S** a proposed signing slot. H bounds +what the previous process *might* have signed; it is not its last actual vote. + +Without a validated clean seal, every vote type and historical local-vote +restoration must obey all of these conditions: + +- **S > H**: never re-sign in the uncertain range during this run, even when + an older history file contains that particular vote. +- **F >= H**: do not sign above the range until verified finality/checkpoint + state has reached its end. F equal to H is sufficient; S equal to H is not. +- **S <= the current acknowledged reservation**, plus all ordinary protocol, + live-joining and configured minimum-slot checks. + +The startup recovery bound stays fixed during the run. Background renewal may +raise the current signing allowance; it does not move the recovery target. +Newly received blocks, elapsed wall time, an RPC tip, replay progress alone, and +`--wait-to-vote-slot` cannot substitute for verified finality/checkpoint state. +The recovery wait can be indefinite if the cluster halts below H. + +For example, suppose votes through 1,015 escaped, detailed history survives only +through 1,012, and the durable reservation is 1,032. After an unclean restart, +slots <= 1,032 remain forbidden. Slot 1,033 is also forbidden while F < 1,032. +Once F >= 1,032, it can pass the recovery gate only after an acknowledged grant +covers 1,033 and all normal voting checks pass. Seeing a new block after restart +does not by itself meet these conditions. + +## Clean shutdown is a durable protocol + +1. Stop/join the voter and leader producer. Halt/join the reservation worker; + drain/join the history writer so an older rename cannot overwrite the seal. +2. Require no latched safety fault or history-write failure, no unresolved + reservation-write uncertainty, and verified recovery through the startup H. +3. Sync exact retained history, including the verified rooted boundary. +4. Sync a reservation record containing the digest of that exact history. + +A successful process exit, a signal handler running, or an attempted final save +is not sufficient. Startup must load and validate the history and reservation, +match the clean digest, and **durably consume the clean marker with a dirty +successor before new vote/leader signing or new history decisions**. A later crash uses +the reservation again, even if the old history still looks valid. + +| Restart state | Vote recovery | Leader recovery | +| --- | --- | --- | +| Valid history and reservation, no matching clean seal | Enforce S > H and F >= H, including restoration. | Enforce S > H and F >= H. | +| Valid matching clean seal, successfully consumed | Resume using exact retained voting decisions and ordinary checks; no additional vote quarantine through H. | Still enforce S > H and F >= H: vote history does not enumerate every block that may have been signed. | +| Missing, corrupt, unreadable or incompatible enrolled state | Refuse automatic reset/startup; do not infer safety from an RPC tip. | Same refusal. | + +Errors during sealing do not authorize treating the session as clean; startup +must validate whichever durable record survived. A failed reservation write +may have reached storage despite its error. Running signers retain only the +previous acknowledged allowance; a later monotonic successful write can resolve +that uncertainty. An unresolved error prevents deliberately sealing clean. + +## Lifetime, storage and enrollment + +The reservation grants up to 32 slots beyond the requested slot and renews when +16 or fewer remain. Only successful file-and-directory sync acknowledgement +publishes new permission. Exhaustion pauses signing while replay/verification +continue. Repeated restarts without new grants do not advance the bound. + +Detailed history uses one ordered writer with one in-flight and at most one +newer pending complete snapshot. New complete snapshots may supersede unwritten +ones. Validation/encoding/signing of the snapshot remain on the voter goroutine; +only file I/O is asynchronous. Writer errors latch a safety fault and stop +voting. An already in-flight vote is still covered by its durable reservation. + +Preserve both `vote_history-.mithril.json` and +`vote_reservation-.mithril.json` independently of AccountsDB snapshots. +The checkpoint encoding cache is only an encoding optimization; it is not the +vote journal or a replacement for the reservation. Rolling back AccountsDB or +an application binary must not roll back either signing-safety file. + +The directory lock only excludes concurrent owners of the same history path. +It cannot fence copies of the identity on other hosts/paths. Signed records and +generation numbers cannot detect an operator restoring an old valid pair of +safety files. Media loss, stale safety-file restoration, dishonest sync behavior +and compromised/copied signing keys are outside this automatic recovery contract. + +Use `--reserved-vote-history` to opt in; default persistence remains synchronous. +First enrollment additionally requires `--initialize-vote-reservation`, exclusive +identity ownership and complete synchronous history from the stopped previous +writer. An empty directory is appropriate only for a previously unused identity. +The software cannot distinguish that case from deleting both files for an old +identity; initialization is not a safe disaster-recovery reset. Remove the +initialization flag afterward. These are CLI flags, not TOML settings. + +Enrolled history uses version 2, rejected by older binaries that do not enforce +the reservation. Missing/corrupt enrolled state or a changed identity, vote +account, authorized voter, genesis or shred version must not be automatically +re-enrolled. Disabling the flag or deleting files is not a supported downgrade. +Transport binding/validation runs before the clean marker is consumed. + +## Live admission is separate from restart recovery + +The live vote admission floor uses retained consensus-pool state and the history +root, not the newest finalization certificate. Replay can still contribute a +notarization within the bounded 16-slot retained tail before later certificates +are collected; finalized retained slots can also receive skips. Durable-root +pruning is ordered behind completed replay events on the voter. These admission +changes also apply to synchronous mode. They do not weaken reserved-mode F >= H. +`--wait-to-vote-slot N` adds an inclusive minimum; it cannot bypass recovery, +execution, retained-root or parent checks. + +## Evidence and limits + +Existing tests map the recovery contract to these cases: + +| Contract | Tests | +| --- | --- | +| Lost valid history suffix; fixed bound across repeated crashes | `TestReservedVotingLostHistorySuffixAndRepeatedCrash` | +| All five vote types, restoration and leader gates at H/H+1 | `TestReservedVotingEverySignatureTypeAtBound` | +| Clean-marker consumption and stricter leader restart | `TestReservedVotingCleanMarkerConsumedBeforeSigning` | +| Digest mismatch, missing/corrupt/domain-mismatched state | `TestReservedCleanDigestMismatchUsesCrashRecovery`, `TestReservedVotingRejectsMissingCorruptOrWrongDomain` | +| Pending/uncertain sync cannot authorize signing | `TestSigningReservationUnacknowledgedSyncCannotAuthorize`, `TestSigningReservationUncertainWriteSurvivesRestart` | +| Writer drain, failure and lost pending snapshots | `TestAsyncHistoryBlockedWriteDoesNotDelayVotesAndCleanCloseDrains`, `TestAsyncHistoryFailureStopsVoterAndPreventsCleanMarker`, `TestAsyncHistoryProcessCrashLosesPendingSnapshots` | + +The subprocess-kill test exercises lost in-flight/pending application snapshots; +it does not power-cycle a host or prove filesystem durability. Fault-injection +and older-history fixtures check the algorithm's decisions under the stated +storage contract. This is not a formal consensus proof or mainnet storage +qualification. Persistence benchmarks likewise do not establish replay or FAST +performance. diff --git a/docs/vote-delivery-persistence-evidence.md b/docs/vote-delivery-persistence-evidence.md new file mode 100644 index 000000000..1ae1f5cfa --- /dev/null +++ b/docs/vote-delivery-persistence-evidence.md @@ -0,0 +1,14 @@ +# Vote Delivery Persistence: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/54b233ff0e27fb929644f7d53bf4a699cb590cd8) +(tag `review-evidence-20260916-vote-delivery-persistence`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/54b233ff0e27fb929644f7d53bf4a699cb590cd8/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/pkg/alpenglow/vote_history.go b/pkg/alpenglow/vote_history.go index d227bbe12..448977fa3 100644 --- a/pkg/alpenglow/vote_history.go +++ b/pkg/alpenglow/vote_history.go @@ -15,6 +15,7 @@ import ( ) const voteHistoryVersion = 1 +const reservedVoteHistoryVersion = 2 var ErrVoteHistoryNotFound = errors.New("alpenglow vote history not found") @@ -23,6 +24,7 @@ var ErrVoteHistoryNotFound = errors.New("alpenglow vote history not found") // Agave when resuming (notarized blocks and ParentReady edges), in addition to // the anti-equivocation vote sets. type VoteHistory struct { + ReservationRequired bool `json:"reservation_required,omitempty"` Version uint32 `json:"version"` NodePubkey solana.PublicKey `json:"node_pubkey"` Root uint64 `json:"root"` @@ -420,50 +422,111 @@ func VoteHistoryFilename(dir string, node solana.PublicKey) string { return filepath.Join(dir, fmt.Sprintf("vote_history-%s.mithril.json", node)) } -// SaveVoteHistory signs the exact serialized history with the validator -// identity and atomically replaces the previous file before a vote can be -// admitted to consensus or sent to the network. +// SaveVoteHistory authenticates and durably replaces the exact history: write, +// file sync, rename, then directory sync. Synchronous-mode callers require +// success before pool admission (which can publish certificates) or network +// enqueue. The BLS signature may already have been computed privately in RAM; +// this is persist-before-publication, not persist-before-BLS-computation. func SaveVoteHistory(dir string, h *VoteHistory, identity ed25519.PrivateKey) error { + return saveVoteHistory(dir, h, identity, true) +} + +// SaveReservedVoteHistory writes and renames without per-vote sync. Success +// does not prove that this history survived a host/power failure. The caller +// must enforce an independently durable signing reservation and its restart +// quarantine; a valid-looking older history is not evidence of completeness. +func SaveReservedVoteHistory(dir string, h *VoteHistory, identity ed25519.PrivateKey) error { + if h == nil || !h.ReservationRequired { + return fmt.Errorf("reserved history requires durable reservation enrollment") + } + return saveVoteHistory(dir, h, identity, false) +} + +// VoteHistorySnapshot owns signed, immutable bytes. It retains no reference to +// the voter's mutable maps or signing key and can be saved by a worker. +type VoteHistorySnapshot struct { + node solana.PublicKey + encoded []byte +} + +// PrepareReservedVoteHistory validates and signs the complete current history +// without filesystem access. Only the owner of h may call this while mutating h. +func PrepareReservedVoteHistory(h *VoteHistory, identity ed25519.PrivateKey) (*VoteHistorySnapshot, error) { + if h == nil || !h.ReservationRequired { + return nil, fmt.Errorf("reserved history requires durable reservation enrollment") + } + encoded, err := encodeVoteHistory(h, identity) + if err != nil { + return nil, err + } + return &VoteHistorySnapshot{node: h.NodePubkey, encoded: encoded}, nil +} + +// SaveReservedVoteHistorySnapshot replaces history without an explicit sync. +// Success means replacement completed, not durable vote acknowledgement. Callers +// must serialize writes and enforce the independent durable reservation. On an +// unclean restart even an intact snapshot cannot bypass the startup bound. +func SaveReservedVoteHistorySnapshot(dir string, snapshot *VoteHistorySnapshot) error { + if snapshot == nil || len(snapshot.encoded) == 0 { + return fmt.Errorf("save reserved history: empty snapshot") + } + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return err + } + return replaceVoteHistoryFile(dir, VoteHistoryFilename(dir, snapshot.node), snapshot.encoded, false) +} + +func saveVoteHistory(dir string, h *VoteHistory, identity ed25519.PrivateKey, durable bool) error { + encoded, err := encodeVoteHistory(h, identity) + if err != nil { + return err + } + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return err + } + return replaceVoteHistoryFile(dir, VoteHistoryFilename(dir, h.NodePubkey), encoded, durable) +} + +func encodeVoteHistory(h *VoteHistory, identity ed25519.PrivateKey) ([]byte, error) { if h == nil { - return fmt.Errorf("save vote history: nil history") + return nil, fmt.Errorf("save vote history: nil history") } if len(identity) != ed25519.PrivateKeySize { - return fmt.Errorf("save vote history: invalid identity key size %d", len(identity)) + return nil, fmt.Errorf("save vote history: invalid identity key size %d", len(identity)) } node := solana.PublicKey(identity.Public().(ed25519.PublicKey)) if node != h.NodePubkey { - return fmt.Errorf("save vote history: identity %s does not match history %s", node, h.NodePubkey) + return nil, fmt.Errorf("save vote history: identity %s does not match history %s", node, h.NodePubkey) } h.Version = voteHistoryVersion + if h.ReservationRequired { + h.Version = reservedVoteHistoryVersion + } if err := h.preparePersistedViews(); err != nil { - return fmt.Errorf("save vote history: %w", err) + return nil, fmt.Errorf("save vote history: %w", err) } defer func() { h.PersistedNotarized = nil h.PersistedParentReady = nil }() if err := h.validatePersistedState(); err != nil { - return fmt.Errorf("save vote history: %w", err) + return nil, fmt.Errorf("save vote history: %w", err) } data, err := json.Marshal(h) if err != nil { - return fmt.Errorf("serialize vote history: %w", err) + return nil, fmt.Errorf("serialize vote history: %w", err) } envelope := savedVoteHistory{ - Version: voteHistoryVersion, + Version: h.Version, Node: node, Data: data, Signature: ed25519.Sign(identity, data), } encoded, err := json.Marshal(envelope) if err != nil { - return fmt.Errorf("serialize saved vote history: %w", err) + return nil, fmt.Errorf("serialize saved vote history: %w", err) } - if err := ensureDurableVoteHistoryDirectory(dir); err != nil { - return err - } - filename := VoteHistoryFilename(dir, node) - return persistVoteHistoryFile(dir, filename, encoded) + return encoded, nil } // ensureDurableVoteHistoryDirectory creates each missing path component and @@ -515,6 +578,10 @@ func ensureDurableVoteHistoryDirectory(dir string) error { // alone does not guarantee that either the bytes or the new directory entry // survives a crash. func persistVoteHistoryFile(dir, filename string, encoded []byte) error { + return replaceVoteHistoryFile(dir, filename, encoded, true) +} + +func replaceVoteHistoryFile(dir, filename string, encoded []byte, durable bool) error { temporary, err := os.CreateTemp(dir, "."+filepath.Base(filename)+".tmp-") if err != nil { return fmt.Errorf("create temporary vote history: %w", err) @@ -538,8 +605,10 @@ func persistVoteHistoryFile(dir, filename string, encoded []byte) error { if n != len(encoded) { return fmt.Errorf("write temporary vote history: %w", io.ErrShortWrite) } - if err := temporary.Sync(); err != nil { - return fmt.Errorf("sync temporary vote history: %w", err) + if durable { + if err := temporary.Sync(); err != nil { + return fmt.Errorf("sync temporary vote history: %w", err) + } } closeErr := temporary.Close() closed = true @@ -551,8 +620,8 @@ func persistVoteHistoryFile(dir, filename string, encoded []byte) error { } renamed = true - if err := syncVoteHistoryDirectory(dir); err != nil { - return err + if durable { + return syncVoteHistoryDirectory(dir) } return nil } @@ -586,7 +655,7 @@ func LoadVoteHistory(dir string, node solana.PublicKey) (*VoteHistory, error) { if err := json.Unmarshal(encoded, &envelope); err != nil { return nil, fmt.Errorf("decode saved vote history: %w", err) } - if envelope.Version != voteHistoryVersion || envelope.Node != node { + if (envelope.Version != voteHistoryVersion && envelope.Version != reservedVoteHistoryVersion) || envelope.Node != node { return nil, fmt.Errorf("saved vote history identity/version mismatch") } if !ed25519.Verify(ed25519.PublicKey(node[:]), envelope.Data, envelope.Signature) { @@ -596,7 +665,7 @@ func LoadVoteHistory(dir string, node solana.PublicKey) (*VoteHistory, error) { if err := json.Unmarshal(envelope.Data, &h); err != nil { return nil, fmt.Errorf("decode vote history: %w", err) } - if h.Version != voteHistoryVersion || h.NodePubkey != node { + if h.Version != envelope.Version || h.NodePubkey != node || h.ReservationRequired != (h.Version == reservedVoteHistoryVersion) { return nil, fmt.Errorf("vote history identity/version mismatch") } if err := h.validatePersistedState(); err != nil { diff --git a/pkg/alpenglow/vote_history_snapshot_test.go b/pkg/alpenglow/vote_history_snapshot_test.go new file mode 100644 index 000000000..540e6fa14 --- /dev/null +++ b/pkg/alpenglow/vote_history_snapshot_test.go @@ -0,0 +1,49 @@ +package alpenglow + +import ( + "crypto/ed25519" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestReservedHistorySnapshotIsImmutable(t *testing.T) { + identity := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + node := solana.PublicKey(identity.Public().(ed25519.PublicKey)) + h := NewVoteHistory(node, 10) + h.ReservationRequired = true + block := BlockID{Slot: 11, Hash: solana.Hash{11}} + require.NoError(t, h.AddVote(NewNotarizationVote(11, block.Hash))) + h.NotarizedBlocks[block] = true + h.AddParentReady(12, block) + snapshot, err := PrepareReservedVoteHistory(h, identity) + require.NoError(t, err) + // Mutate/prune every transport-backed collection after taking the snapshot. + h.SetRoot(20) + require.NoError(t, h.AddVote(NewSkipVote(21))) + for i := range identity { + identity[i] = 0 + } + dir := t.TempDir() + require.NoError(t, SaveReservedVoteHistorySnapshot(dir, snapshot)) + loaded, err := LoadVoteHistory(dir, node) + require.NoError(t, err) + require.Equal(t, uint64(10), loaded.Root) + require.True(t, loaded.VotedAt(11)) + require.True(t, loaded.IsBlockNotarized(block)) + require.True(t, loaded.IsParentReady(12, block)) + require.False(t, loaded.HasSkipped(21)) +} + +func TestReservedHistorySnapshotRequiresEnrollmentAndValidHistory(t *testing.T) { + identity := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + h := NewVoteHistory(solana.PublicKey(identity.Public().(ed25519.PublicKey)), 10) + _, err := PrepareReservedVoteHistory(h, identity) + require.Error(t, err) + h.ReservationRequired = true + h.Voted[11] = true // Inconsistent with canonical VotesCast. + _, err = PrepareReservedVoteHistory(h, identity) + require.Error(t, err) + require.Error(t, SaveReservedVoteHistorySnapshot(t.TempDir(), &VoteHistorySnapshot{})) +} diff --git a/pkg/alpenglow/vote_reservation.go b/pkg/alpenglow/vote_reservation.go new file mode 100644 index 000000000..b577da189 --- /dev/null +++ b/pkg/alpenglow/vote_reservation.go @@ -0,0 +1,104 @@ +package alpenglow + +import ( + "crypto/ed25519" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/gagliardetto/solana-go" + "golang.org/x/sys/unix" +) + +// VoteReservation is the durable upper bound on slots this identity may sign, +// not a record of slots actually signed. It must survive independently of +// AccountsDB checkpoints and must never be restored from an older snapshot. +// A signature authenticates this file; it does not prove freshness against +// rollback, nor does Generation provide an external monotonic counter. +// CleanHistoryDigest permits exact-history vote recovery only after validation +// and durable consumption by a dirty successor before signing. It does not +// certify complete leader-block history. See docs/reserved-vote-history.md. +type VoteReservation struct { + Version uint32 `json:"version"` + Node solana.PublicKey `json:"node"` + VoteAccount solana.PublicKey `json:"vote_account"` + AuthorizedVoter solana.PublicKey `json:"authorized_voter"` + Genesis solana.Hash `json:"genesis"` + ShredVersion uint16 `json:"shred_version"` + Generation uint64 `json:"generation"` + Through uint64 `json:"through"` + CleanHistoryDigest []byte `json:"clean_history_digest,omitempty"` +} + +func VoteReservationFilename(dir string, node solana.PublicKey) string { + return filepath.Join(dir, fmt.Sprintf("vote_reservation-%s.mithril.json", node)) +} + +// LockVoteHistory excludes concurrent owners of this directory. Operators must +// still fence copies of the same identity on other hosts or in other paths. +func LockVoteHistory(dir string, node solana.PublicKey) (*os.File, error) { + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return nil, err + } + f, err := os.OpenFile(filepath.Join(dir, ".vote_history-"+node.String()+".lock"), os.O_CREATE|os.O_RDWR, 0600) + if err != nil { + return nil, err + } + if err := unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB); err != nil { + f.Close() + return nil, fmt.Errorf("vote history already owned: %w", err) + } + return f, nil +} + +func LoadVoteReservation(dir string, node solana.PublicKey) (VoteReservation, error) { + var r VoteReservation + encoded, err := os.ReadFile(VoteReservationFilename(dir, node)) + if err != nil { + return r, err + } + var envelope savedVoteHistory + if err := json.Unmarshal(encoded, &envelope); err != nil { + return r, err + } + if envelope.Version != 1 || envelope.Node != node || !ed25519.Verify(ed25519.PublicKey(node[:]), envelope.Data, envelope.Signature) { + return r, errors.New("invalid vote reservation signature/version/identity") + } + if err := json.Unmarshal(envelope.Data, &r); err != nil { + return r, err + } + if r.Version != 1 || r.Node != node || r.Generation == 0 || (len(r.CleanHistoryDigest) != 0 && len(r.CleanHistoryDigest) != sha256.Size) { + return r, errors.New("invalid vote reservation record") + } + return r, nil +} + +func SaveVoteReservation(dir string, r VoteReservation, identity ed25519.PrivateKey) error { + if len(identity) != ed25519.PrivateKeySize || solana.PublicKey(identity.Public().(ed25519.PublicKey)) != r.Node || r.Version != 1 || r.Generation == 0 { + return errors.New("invalid vote reservation signer/record") + } + data, err := json.Marshal(r) + if err != nil { + return err + } + encoded, err := json.Marshal(savedVoteHistory{Version: 1, Node: r.Node, Data: data, Signature: ed25519.Sign(identity, data)}) + if err != nil { + return err + } + if err := ensureDurableVoteHistoryDirectory(dir); err != nil { + return err + } + return persistVoteHistoryFile(dir, VoteReservationFilename(dir, r.Node), encoded) +} + +func VoteHistoryDigest(dir string, node solana.PublicKey) ([]byte, error) { + data, err := os.ReadFile(VoteHistoryFilename(dir, node)) + if err != nil { + return nil, err + } + digest := sha256.Sum256(data) + return digest[:], nil +} diff --git a/pkg/alpenglow/vote_reservation_test.go b/pkg/alpenglow/vote_reservation_test.go new file mode 100644 index 000000000..9e1840ade --- /dev/null +++ b/pkg/alpenglow/vote_reservation_test.go @@ -0,0 +1,71 @@ +package alpenglow + +import ( + "crypto/ed25519" + "encoding/json" + "os" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestReservedHistoryFormatAndIntegrity(t *testing.T) { + key := ed25519.NewKeyFromSeed(make([]byte, 32)) + node := solana.PublicKey(key.Public().(ed25519.PublicKey)) + dir := t.TempDir() + h := NewVoteHistory(node, 39) + require.Error(t, SaveReservedVoteHistory(dir, h, key)) + h.ReservationRequired = true + require.NoError(t, h.AddVote(NewSkipVote(44))) + require.NoError(t, SaveReservedVoteHistory(dir, h, key)) + raw, err := os.ReadFile(VoteHistoryFilename(dir, node)) + require.NoError(t, err) + var envelope savedVoteHistory + require.NoError(t, json.Unmarshal(raw, &envelope)) + require.Equal(t, uint32(2), envelope.Version, "legacy reader must reject hybrid history") + loaded, err := LoadVoteHistory(dir, node) + require.NoError(t, err) + require.True(t, loaded.HasSkipped(44)) + require.True(t, loaded.ReservationRequired) + envelope.Data[10] ^= 1 + raw, err = json.Marshal(envelope) + require.NoError(t, err) + require.NoError(t, os.WriteFile(VoteHistoryFilename(dir, node), raw, 0600)) + _, err = LoadVoteHistory(dir, node) + require.Error(t, err) +} + +func BenchmarkVoteHistoryPersistence(b *testing.B) { + key := ed25519.NewKeyFromSeed(make([]byte, 32)) + node := solana.PublicKey(key.Public().(ed25519.PublicKey)) + for _, reserved := range []bool{false, true} { + name := "synchronous" + if reserved { + name = "reserved-write-rename" + } + b.Run(name, func(b *testing.B) { + dir := b.TempDir() + h := NewVoteHistory(node, 39) + h.ReservationRequired = reserved + for slot := uint64(40); slot < 72; slot++ { + if err := h.AddVote(NewNotarizationVote(slot, solana.Hash{byte(slot)})); err != nil { + b.Fatal(err) + } + } + save := SaveVoteHistory + if reserved { + save = SaveReservedVoteHistory + } + if err := SaveVoteHistory(dir, h, key); err != nil { + b.Fatal(err) + } + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := save(dir, h, key); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/blockprod/leader.go b/pkg/blockprod/leader.go index e304ffb8d..65c062173 100644 --- a/pkg/blockprod/leader.go +++ b/pkg/blockprod/leader.go @@ -97,6 +97,7 @@ type LeaderLoop struct { alpenglowClock bool parentContext func(uint64) ParentContext productionParent func(uint64) alpenglow.BlockProductionParent + canSignSlot func(uint64) bool onBlock func(*b.Block) commitLeaderSlot func(replay.CommitLeaderInput) (*sealevel.SlotCtx, error) @@ -145,6 +146,7 @@ type LeaderLoopConfig struct { AlpenglowClock bool ParentContext func(uint64) ParentContext ProductionParent func(slot uint64) alpenglow.BlockProductionParent + CanSignSlot func(slot uint64) bool OnBlock func(*b.Block) CurrentSlot func() uint64 LeaderForSlot func(uint64) (solana.PublicKey, bool) @@ -181,6 +183,7 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { alpenglowClock: cfg.AlpenglowClock, parentContext: cfg.ParentContext, productionParent: cfg.ProductionParent, + canSignSlot: cfg.CanSignSlot, onBlock: cfg.OnBlock, currentSlot: cfg.CurrentSlot, leaderForSlot: cfg.LeaderForSlot, @@ -1101,6 +1104,9 @@ func (l *LeaderLoop) revalidateProductionParentForStartLocked(slot uint64, selec } func (l *LeaderLoop) startSlotLocked(slot uint64) error { + if l.canSignSlot != nil && !l.canSignSlot(slot) { + return fmt.Errorf("%w: waiting for durable signing reservation", errParentNotReady) + } selectedParent, parentReadyRequired, err := l.resolveProductionParent(slot) if err != nil { return err diff --git a/pkg/blockprod/leader_signing_reservation_test.go b/pkg/blockprod/leader_signing_reservation_test.go new file mode 100644 index 000000000..4b7e09c10 --- /dev/null +++ b/pkg/blockprod/leader_signing_reservation_test.go @@ -0,0 +1,17 @@ +package blockprod + +import ( + "github.com/stretchr/testify/require" + "testing" +) + +func TestSigningReservationGatesEveryLeaderSlotBeforeBuild(t *testing.T) { + for slot := uint64(40); slot < 44; slot++ { + var checked uint64 + l := &LeaderLoop{canSignSlot: func(s uint64) bool { checked = s; return false }} + require.ErrorIs(t, l.startSlotLocked(slot), errParentNotReady) + require.Equal(t, slot, checked) + // All other builder dependencies are deliberately nil: rejection must happen + // before accessing a working bank, executing or signing any shreds. + } +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e7416c90..c31bf7448 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -25,6 +25,7 @@ func ApplyDefaults(v *viper.Viper) { v.SetDefault("validator.tpu_quic_bind_addr", "") v.SetDefault("validator.advertised_ip", "") v.SetDefault("validator.tpu_sigverify_workers", 0) + v.SetDefault("validator.wait_to_vote_slot", uint64(0)) } // LedgerConfig holds ledger-related configuration (matches Firedancer [ledger] section) @@ -235,6 +236,7 @@ type ValidatorConfig struct { TPUQUICBindAddr string `toml:"tpu_quic_bind_addr" mapstructure:"tpu_quic_bind_addr"` AdvertisedIP string `toml:"advertised_ip" mapstructure:"advertised_ip"` TPUSigverifyWorkers int `toml:"tpu_sigverify_workers" mapstructure:"tpu_sigverify_workers"` + WaitToVoteSlot uint64 `toml:"wait_to_vote_slot" mapstructure:"wait_to_vote_slot"` // Minimum slot for new votes; automatic startup cutoff still applies } // Config holds all configuration options for Mithril (Firedancer-style hierarchy) diff --git a/pkg/consensus/engine.go b/pkg/consensus/engine.go index f57cb2ab5..5cf2cb203 100644 --- a/pkg/consensus/engine.go +++ b/pkg/consensus/engine.go @@ -389,7 +389,7 @@ func (e *AlpenglowObserverEngine) EnableVoting(cfg VotingConfig) error { // events, matching Agave's initial_parent_ready selection. if slot, parent, ok := voter.history.HighestParentReadyMatching(func(parent alpenglow.BlockID) bool { return !e.ensureChain().IsObjectivelyInvalidBlock(parent) - }); ok && slot > root.Slot { + }); ok && slot > root.Slot && (voter.reservation == nil || voter.reservation.recoverThrough == 0) { if !e.ensurePool().RestoreParentReady(slot, parent) { mlog.Log.FileOnlyf("ALPENGLOW voting: ignored persisted ParentReady slot=%d parent=%s because newer root/live tracker state is authoritative", slot, parent) } @@ -975,11 +975,18 @@ func (e *AlpenglowObserverEngine) injectLocalVote(message alpenglow.VoteMessage, } } -// alpenglowVoteActionFloor is the highest slot on which this validator must -// not initiate a new vote. The pool root is its strict admission boundary; -// direct finality is included because the pool deliberately retains a short -// reward-accounting tail behind finality where network votes remain useful. +// alpenglowVoteActionFloor is the retained pool's strict admission boundary. +// Network finality is not a voting root: a replayed block may still contribute +// a notarization to a later fast certificate and its slot+8 reward certificate. +// The voter also checks its own persisted history root before signing. func (e *AlpenglowObserverEngine) alpenglowVoteActionFloor() uint64 { + return e.ensurePool().Snapshot().RootSlot +} + +// alpenglowVerifiedFinalityFloor releases crash-recovery reservations. Keep this +// independent of live vote admission: a retained reward window must not weaken +// the requirement to pass every slot that may have been signed before a crash. +func (e *AlpenglowObserverEngine) alpenglowVerifiedFinalityFloor() uint64 { floor := e.ensurePool().Snapshot().RootSlot if finalized := e.ensureChain().Snapshot().LatestDirectFinalizedBlock.Slot; finalized > floor { floor = finalized @@ -1542,6 +1549,25 @@ func (e *AlpenglowObserverEngine) PruneAlpenglowBefore(slot uint64) { if slot == 0 { return } + // Replay enqueues its completed-block event before publishing a durable + // promotion. Retire the pool, execution proof and history on that same + // ordered voter stream, so a fast checkpoint cannot overtake the vote. + e.voterMu.RLock() + voter := e.voter + e.voterMu.RUnlock() + if voter != nil { + if err := voter.enqueue(voterEvent{kind: voterEventDurableRoot, slot: slot}); err != nil { + e.latchSafetyError(err) + } + return + } + e.applyAlpenglowDurableRoot(slot) +} + +// applyAlpenglowDurableRoot is called by the voter after earlier replay events, +// or synchronously by an observer without a voting loop. Startup root restore +// remains a separate, immediate barrier in SetAlpenglowRoot. +func (e *AlpenglowObserverEngine) applyAlpenglowDurableRoot(slot uint64) alpenglow.BlockID { e.poolOutputMu.Lock() defer e.poolOutputMu.Unlock() @@ -1557,13 +1583,11 @@ func (e *AlpenglowObserverEngine) PruneAlpenglowBefore(slot uint64) { if e.certPool != nil { e.certPool.ObserveFloor(slot) } - if err := e.enqueueVoter(voterEvent{kind: voterEventRoot, root: root}); err != nil { - e.latchSafetyError(err) - } // Replay calls this only after the fold through slot is durably committed. // Keep the transport peer window tied to that local root, not to speculative // certificate finality or the pool's reward-retention floor. e.advanceVotorPeerRoot(slot) + return root } func (e *AlpenglowObserverEngine) pruneInvalidBlockIDsBefore(slot uint64) { diff --git a/pkg/consensus/vote_history_writer.go b/pkg/consensus/vote_history_writer.go new file mode 100644 index 000000000..9d175d6d4 --- /dev/null +++ b/pkg/consensus/vote_history_writer.go @@ -0,0 +1,125 @@ +package consensus + +import ( + "errors" + "fmt" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" +) + +// One serial writer, one in-flight snapshot and at most one newer pending +// snapshot. A newer complete history supersedes an unwritten snapshot; every +// retained, unrooted voting decision is still present in that newer history. +// The independently durable reservation, not this queue, authorizes signing. +// In-flight/pending snapshots may be lost on process death; even a completed +// unsynced replacement may be lost on host/power failure. Neither submitted nor +// written is a durable vote acknowledgement. Recovery must use the startup +// reservation unless the separate clean-history seal validates. +type voteHistoryWriter struct { + mu sync.Mutex + pending *alpenglow.VoteHistorySnapshot + closing bool + failure error + submitted uint64 + written uint64 + coalesced uint64 + wake chan struct{} + done chan struct{} + persist func(*alpenglow.VoteHistorySnapshot) error + onError func(error) +} + +func newVoteHistoryWriter(persist func(*alpenglow.VoteHistorySnapshot) error, onError func(error)) *voteHistoryWriter { + w := &voteHistoryWriter{wake: make(chan struct{}, 1), done: make(chan struct{}), persist: persist, onError: onError} + go w.run() + return w +} + +// submit does no I/O and never waits for the writer. The mutex only protects +// pointer/counter changes; neither persistence nor error callbacks hold it. +// A nil return means queued only. It must never replace the reservation check. +func (w *voteHistoryWriter) submit(snapshot *alpenglow.VoteHistorySnapshot) error { + if snapshot == nil { + return errors.New("nil vote-history snapshot") + } + w.mu.Lock() + if w.failure != nil { + err := w.failure + w.mu.Unlock() + return err + } + if w.closing { + w.mu.Unlock() + return errors.New("vote-history writer is closed") + } + if w.pending != nil { + w.coalesced++ + } + w.pending = snapshot + w.submitted++ + w.mu.Unlock() + w.notify() + return nil +} + +func (w *voteHistoryWriter) notify() { + select { + case w.wake <- struct{}{}: + default: + } +} + +func (w *voteHistoryWriter) run() { + defer close(w.done) + for range w.wake { + for { + w.mu.Lock() + snapshot := w.pending + w.pending = nil + closing := w.closing + w.mu.Unlock() + if snapshot == nil { + if closing { + return + } + break + } + if err := w.persist(snapshot); err != nil { + err = fmt.Errorf("background vote-history write: %w", err) + w.mu.Lock() + w.failure = err + w.pending = nil + w.closing = true + w.mu.Unlock() + if w.onError != nil { + w.onError(err) + } + return + } + w.mu.Lock() + w.written++ + w.mu.Unlock() + } + } +} + +// close rejects new submissions and drains every retained snapshot. The voter +// must join this worker before writing and syncing its final clean history, +// otherwise an older in-flight rename could overwrite the sealed history. +func (w *voteHistoryWriter) close() error { + w.mu.Lock() + w.closing = true + w.mu.Unlock() + w.notify() + <-w.done + w.mu.Lock() + defer w.mu.Unlock() + return w.failure +} + +func (w *voteHistoryWriter) counters() (submitted, written, coalesced uint64) { + w.mu.Lock() + defer w.mu.Unlock() + return w.submitted, w.written, w.coalesced +} diff --git a/pkg/consensus/vote_history_writer_test.go b/pkg/consensus/vote_history_writer_test.go new file mode 100644 index 000000000..74c7bbfcd --- /dev/null +++ b/pkg/consensus/vote_history_writer_test.go @@ -0,0 +1,201 @@ +package consensus + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestAsyncHistoryBlockedWriteDoesNotDelayVotesAndCleanCloseDrains(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + require.NoError(t, v.historyWriter.close()) + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + t.Cleanup(unblock) + first := true // Owned only by the serial writer. + v.historyWriter = newVoteHistoryWriter(func(s *alpenglow.VoteHistorySnapshot) error { + if first { + first = false + close(entered) + <-release + } + return alpenglow.SaveReservedVoteHistorySnapshot(cfg.HistoryDir, s) + }, nil) + voted, err := v.cast(alpenglow.NewSkipVote(44), false) + require.NoError(t, err) + require.True(t, voted) + <-entered + castDone := make(chan error, 1) + go func() { + for _, slot := range []uint64{45, 46} { + ok, err := v.cast(alpenglow.NewSkipVote(slot), false) + if err != nil || !ok { + castDone <- errors.New("vote failed while history writer was blocked") + return + } + } + castDone <- nil + }() + select { + case err := <-castDone: + require.NoError(t, err) + case <-time.After(time.Second): + t.Fatal("disk writer blocked voting") + } + submitted, written, coalesced := v.historyWriter.counters() + require.Equal(t, uint64(3), submitted) + require.Zero(t, written) + require.Equal(t, uint64(1), coalesced) + onDisk, err := alpenglow.LoadVoteHistory(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.False(t, onDisk.HasSkipped(44), "I/O should still be blocked") + // Even when disk history lags, complete in-memory decisions forbid conflict. + voted, err = v.cast(alpenglow.NewNotarizationVote(44, solana.Hash{7}), false) + require.NoError(t, err) + require.False(t, voted) + closed := make(chan error, 1) + go func() { closed <- v.close() }() + require.Eventually(t, func() bool { + v.historyWriter.mu.Lock() + defer v.historyWriter.mu.Unlock() + return v.historyWriter.closing + }, time.Second, time.Millisecond) + select { + case err := <-closed: + t.Fatalf("close returned before draining its writer: %v", err) + default: + } + r, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) + unblock() + require.NoError(t, <-closed) + onDisk, err = alpenglow.LoadVoteHistory(cfg.HistoryDir, v.node) + require.NoError(t, err) + for _, slot := range []uint64{44, 45, 46} { + require.True(t, onDisk.HasSkipped(slot)) + } + r, err = alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + digest, err := alpenglow.VoteHistoryDigest(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Equal(t, digest, r.CleanHistoryDigest) + _, written, _ = v.historyWriter.counters() + require.Equal(t, uint64(2), written, "old in-flight snapshot must finish before newest complete snapshot") +} + +func TestAsyncHistoryFailureIsStickyAndReportedWithoutMoreSubmissions(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + h := alpenglow.NewVoteHistory(voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount).Validators[0].NodePubkey, 39) + h.ReservationRequired = true + snapshot, err := alpenglow.PrepareReservedVoteHistory(h, cfg.Identity) + require.NoError(t, err) + diskErr := errors.New("injected disk failure") + reported := make(chan error, 1) + w := newVoteHistoryWriter(func(*alpenglow.VoteHistorySnapshot) error { return diskErr }, func(err error) { reported <- err }) + require.NoError(t, w.submit(snapshot)) + select { + case err := <-reported: + require.ErrorIs(t, err, diskErr) + case <-time.After(time.Second): + t.Fatal("background failure was not reported") + } + require.ErrorIs(t, w.submit(snapshot), diskErr) + require.ErrorIs(t, w.close(), diskErr) + require.ErrorIs(t, w.close(), diskErr) +} + +func TestAsyncHistoryFailureStopsVoterAndPreventsCleanMarker(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + e, err := NewEngine(Config{AlpenglowIdentity: cfg.Identity, AlpenglowShredVersion: 0x1234}) + require.NoError(t, err) + t.Cleanup(func() { _ = e.Close() }) + set := voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount) + e.SetAlpenglowEpochLookup(cfg.EpochForSlot) + require.NoError(t, e.SetAlpenglowValidatorSet(set)) + root := alpenglow.BlockID{Slot: 39, Hash: solana.Hash{39}} + e.SetAlpenglowRoot(root) + v, err := newAlpenglowVoterUnstarted(e, cfg, root, []alpenglow.ValidatorSet{set}) + require.NoError(t, err) + t.Cleanup(func() { _ = v.close() }) + reserveThrough(t, v.reservation, 44) + require.NoError(t, v.historyWriter.close()) + diskErr := errors.New("injected background disk failure") + v.historyWriter = newVoteHistoryWriter(func(*alpenglow.VoteHistorySnapshot) error { return diskErr }, v.failHistoryWrite) + require.NoError(t, v.history.AddVote(alpenglow.NewSkipVote(44))) + require.NoError(t, v.saveHistory()) + select { + case <-v.done: + case <-time.After(time.Second): + t.Fatal("disk failure did not stop the voter") + } + require.ErrorIs(t, e.safetyError(), diskErr) + _, _, err = v.sign(alpenglow.NewSkipVote(45), false) + require.ErrorIs(t, err, diskErr) + require.Error(t, v.enqueue(voterEvent{kind: voterEventBlockTimeout, slot: 45})) + require.ErrorIs(t, v.close(), diskErr) + r, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) +} + +// Kill a subprocess while its first history write is blocked and a newer +// snapshot is pending. Successful earlier reservation syncs survive this +// process crash; this is deliberately not a host power-loss test. +func TestAsyncHistoryProcessCrashLosesPendingSnapshots(t *testing.T) { + const childEnv = "MITHRIL_ASYNC_HISTORY_TEST_DIR" + if dir := os.Getenv(childEnv); dir != "" { + cfg := reservedTestConfig(dir) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + require.NoError(t, v.historyWriter.close()) + entered := make(chan struct{}) + v.historyWriter = newVoteHistoryWriter(func(*alpenglow.VoteHistorySnapshot) error { + close(entered) + select {} + }, nil) + for _, slot := range []uint64{44, 45} { + voted, err := v.cast(alpenglow.NewSkipVote(slot), false) + require.NoError(t, err) + require.True(t, voted) + if slot == 44 { + <-entered + } + } + require.NoError(t, os.WriteFile(filepath.Join(dir, "ready"), []byte("ready"), 0600)) + select {} + } + dir := t.TempDir() + cmd := exec.Command(os.Args[0], "-test.run=^TestAsyncHistoryProcessCrashLosesPendingSnapshots$", "-test.count=1") + cmd.Env = append(os.Environ(), childEnv+"="+dir) + require.NoError(t, cmd.Start()) + t.Cleanup(func() { _ = cmd.Process.Kill() }) + require.Eventually(t, func() bool { _, err := os.Stat(filepath.Join(dir, "ready")); return err == nil }, 10*time.Second, time.Millisecond) + require.NoError(t, cmd.Process.Kill()) + require.Error(t, cmd.Wait()) + cfg := reservedTestConfig(dir) + cfg.InitializeVoteReservation = false + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.False(t, v.history.HasSkipped(44)) + require.False(t, v.history.HasSkipped(45)) + h := v.reservation.recoverThrough + require.GreaterOrEqual(t, h, uint64(45)) + _, _, err = v.sign(alpenglow.NewSkipVote(44), false) + require.ErrorIs(t, err, errVoterNotReady) + _, _, err = v.sign(alpenglow.NewSkipVote(h+1), false) + require.ErrorIs(t, err, errVoterNotReady, "verified finality must reach the lost history's bound") +} diff --git a/pkg/consensus/vote_reservation.go b/pkg/consensus/vote_reservation.go new file mode 100644 index 000000000..564cfc29d --- /dev/null +++ b/pkg/consensus/vote_reservation.go @@ -0,0 +1,296 @@ +package consensus + +import ( + "bytes" + "crypto/ed25519" + "errors" + "fmt" + "math" + "os" + "sync" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/gagliardetto/solana-go" +) + +const signingReserveSlots = uint64(32) +const signingRenewRemaining = uint64(16) + +// signingReservation bounds what a crash may erase from detailed vote history. +// Intended guarantee: losing recent history must not authorize conflicting +// voting/leader actions after restart. It does NOT guarantee that every vote +// survives on disk, immediate restart voting, or recovery from safety-file rollback. +// +// On an enrolled restart, H is the startup reservation. Without a clean-history +// seal, all vote types (including restored votes) are forbidden at slots <= H; +// slots > H also wait until verified finality/checkpoint state reaches H. +// Leaders always obey that startup barrier, even after a clean vote-history seal. +// The current acknowledged Through separately caps every new signing permission. +// Renewal can raise Through, but never moves this run's fixed recovery barrier. +// +// The worker owns record after startup. Only a successful file+directory sync +// publishes Through to signers; a request, queued write or uncertain sync cannot. +// This assumes storage honors sync and a single fenced identity owner preserves +// the current reservation independently of AccountsDB. See docs/reserved-vote-history.md. +type signingReservation struct { + uncertain bool // Worker only, read after halt. Failed sync may have reached storage. + record alpenglow.VoteReservation + through atomic.Uint64 + desired atomic.Uint64 + stopped atomic.Bool + recoverThrough uint64 // Startup H, or zero after first enrollment / a validated clean-history seal. + leaderThrough uint64 // Exact leader production history is not saved: always skip the old range. + wake chan struct{} + changed chan struct{} + stop chan struct{} + done chan struct{} + stopOnce sync.Once + persist func(alpenglow.VoteReservation) error +} + +func openSigningReservation(cfg VotingConfig, node solana.PublicKey, shredVersion uint16, history *alpenglow.VoteHistory) (*signingReservation, error) { + if cfg.Genesis == (solana.Hash{}) { + return nil, errors.New("reserved voting requires the bound genesis hash") + } + expected := alpenglow.VoteReservation{Version: 1, Node: node, VoteAccount: cfg.VoteAccount, AuthorizedVoter: solana.PublicKey(cfg.AuthorizedVoter.Public().(ed25519.PublicKey)), Genesis: cfg.Genesis, ShredVersion: shredVersion} + record, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, node) + initializing := errors.Is(err, os.ErrNotExist) + if initializing { + if !cfg.InitializeVoteReservation || history.ReservationRequired { + return nil, errors.New("missing vote reservation; explicit first enrollment with complete synchronous history is required") + } + record = expected + record.Generation = 1 + record.Through = history.Root + for slot := range history.VotesCast { + record.Through = max(record.Through, slot) + } + // The baseline is made durable before the first reservation is created. + if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { + return nil, err + } + } else if err != nil { + return nil, fmt.Errorf("refuse unsafe reservation reset: %w", err) + } + if record.Node != expected.Node || record.VoteAccount != expected.VoteAccount || record.AuthorizedVoter != expected.AuthorizedVoter || record.Genesis != expected.Genesis || record.ShredVersion != expected.ShredVersion { + return nil, errors.New("vote reservation cluster or signing identity mismatch; explicit domain migration is required") + } + if record.Through == math.MaxUint64 || record.Generation == math.MaxUint64 { + return nil, errors.New("vote reservation exhausted") + } + for slot := range history.VotesCast { + if slot > record.Through { + return nil, fmt.Errorf("history slot %d exceeds durable reservation %d", slot, record.Through) + } + } + r := &signingReservation{record: record, recoverThrough: record.Through, leaderThrough: record.Through, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + r.persist = func(next alpenglow.VoteReservation) error { + return alpenglow.SaveVoteReservation(cfg.HistoryDir, next, cfg.Identity) + } + if initializing { + r.recoverThrough = 0 + } else if len(record.CleanHistoryDigest) != 0 { + digest, err := alpenglow.VoteHistoryDigest(cfg.HistoryDir, node) + if err != nil { + return nil, err + } + if bytes.Equal(digest, record.CleanHistoryDigest) { + r.recoverThrough = 0 + } + } + // A matching digest proves exact history only for the sealed session. Consume + // that exception with a durably acknowledged dirty successor before allowing + // new vote/leader signing or new detailed-history decisions. A crash after + // this write must use H, + // even if the detailed history file still looks valid or matches the old seal. + r.record.CleanHistoryDigest = nil + r.record.Generation++ + if err := r.persist(r.record); err != nil { + return nil, fmt.Errorf("consume vote reservation session: %w", err) + } + history.ReservationRequired = true + // Version 2 is deliberately rejected by older binaries that do not enforce H. + if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { + return nil, err + } + r.through.Store(r.record.Through) + mlog.Log.Infof("ALPENGLOW signing reservation: through=%d recovery_through=%d leader_recovery_through=%d", r.record.Through, r.recoverThrough, r.leaderThrough) + go r.run() + return r, nil +} + +// allow is nonblocking and protects vote signing/restoration and leader slots. +// For a nonzero startup barrier H, require slot > H AND finalized >= H. Merely +// observing a new block, waiting elapsed time, or replaying past H is not proof +// that prior decisions can be forgotten. Finality comes from verified consensus/ +// checkpoint state, never an RPC tip; --wait-to-vote-slot cannot override it. +// Passing this gate is necessary, not sufficient: normal protocol checks apply. +func (r *signingReservation) allow(slot, finalized uint64, leader bool) bool { + if r.stopped.Load() { + return false + } + floor := r.recoverThrough + if leader { + floor = r.leaderThrough + } + if floor != 0 && (slot <= floor || finalized < floor) { + return false + } + r.request(slot) + return slot <= r.through.Load() +} + +func (r *signingReservation) request(slot uint64) { + for { + old := r.desired.Load() + if slot <= old || r.desired.CompareAndSwap(old, slot) { + break + } + } + through := r.through.Load() + if slot > through || through-slot <= signingRenewRemaining { + select { + case r.wake <- struct{}{}: + default: + } + } +} + +func (r *signingReservation) run() { + defer close(r.done) + retry := time.NewTicker(250 * time.Millisecond) + defer retry.Stop() + var warned bool + for { + select { + case <-r.stop: + return + case <-r.wake: + case <-retry.C: + } + if r.stopped.Load() { + return + } + slot := r.desired.Load() + if slot == 0 || (slot <= r.record.Through && r.record.Through-slot > signingRenewRemaining) { + continue + } + if slot > math.MaxUint64-signingReserveSlots || r.record.Generation == math.MaxUint64 { + continue + } // Never wrap or grant permission. + next := r.record + next.Through = max(next.Through, slot+signingReserveSlots) + next.Generation++ + next.CleanHistoryDigest = nil + if err := r.persist(next); err != nil { + r.uncertain = true + if !warned { + mlog.Log.Errorf("ALPENGLOW signing reservation renewal failed; permission remains through %d: %v", r.through.Load(), err) + warned = true + } + continue // Retry the same or a greater bound; an uncertain sync never grants permission. + } + warned = false + r.uncertain = false + r.record = next + r.through.Store(next.Through) + select { + case r.changed <- struct{}{}: + default: + } + } +} + +func (r *signingReservation) halt() { + r.stopOnce.Do(func() { r.stopped.Store(true); close(r.stop) }) + <-r.done +} + +// seal may be called only after the voter loop and leader producer have stopped, +// and after the ordered history writer has been drained/joined. The caller must +// also establish verified finality >= recoverThrough and no latched safety fault. +// Sync exact history first, then sync its digest in the reservation. A normal +// process exit or successful unsynced rename alone is not a clean seal. On an +// error the caller must not assume cleanliness; restart validates whichever +// durable record survived. The seal never relaxes the next run's leader barrier. +func (r *signingReservation) seal(dir string, history *alpenglow.VoteHistory, identity ed25519.PrivateKey) error { + r.halt() + if r.uncertain { + return errors.New("uncertain reservation write; retaining unclean recovery") + } + if err := alpenglow.SaveVoteHistory(dir, history, identity); err != nil { + return err + } + digest, err := alpenglow.VoteHistoryDigest(dir, history.NodePubkey) + if err != nil { + return err + } + if r.record.Generation == math.MaxUint64 { + return errors.New("vote reservation generation exhausted") + } + next := r.record + next.Generation++ + next.CleanHistoryDigest = digest + return r.persist(next) +} + +// Retain only events blocked on renewal, not historical catch-up traffic. +// Replay the original event after acknowledgement so normal finality, parent, +// execution and invalidation checks still decide whether to vote. +func (v *alpenglowVoter) retainReservationEvent(event voterEvent) { + r := v.reservation + if r == nil { + return + } + var slot uint64 + switch event.kind { + case voterEventBlock: + slot = event.block.Block.Slot + case voterEventBlockTimeout, voterEventCrashedLeaderTimeout: + slot = event.slot | (alpenglow.LeaderWindowSlots - 1) + case voterEventConsensus: + switch event.consensus.Kind { + case alpenglow.ConsensusEventBlockNotarized, alpenglow.ConsensusEventParentReady, alpenglow.ConsensusEventSafeToNotar, alpenglow.ConsensusEventSafeToSkip: + slot = event.consensus.Slot + if event.consensus.Kind == alpenglow.ConsensusEventSafeToNotar || event.consensus.Kind == alpenglow.ConsensusEventSafeToSkip { + slot |= alpenglow.LeaderWindowSlots - 1 + } + default: + return + } + default: + return + } + if slot <= r.through.Load() || slot <= v.admissionFloor() || slot < v.waitToVoteSlot || v.engine.alpenglowVerifiedFinalityFloor() < r.recoverThrough || slot <= r.recoverThrough { + return + } + if !v.votingStarted && v.readyToVote != nil && !v.readyToVote(slot) { + return + } + r.request(slot) + if len(v.reservationEvents) < votorEventQueueSize { + v.reservationEvents = append(v.reservationEvents, event) + } +} + +// AlpenglowCanSignLeaderSlot protects every produced slot, including the +// trailing slots of a leader window. A clean vote-history marker is not a +// complete leader-block history, so leaders always skip the old reservation. +func (e *AlpenglowObserverEngine) AlpenglowCanSignLeaderSlot(slot uint64) bool { + if e.safetyError() != nil { + return false + } + e.voterMu.RLock() + defer e.voterMu.RUnlock() + v := e.voter + if v == nil { + return false + } + if v.reservation == nil { + return true + } + return v.reservation.allow(slot, e.alpenglowVerifiedFinalityFloor(), true) +} diff --git a/pkg/consensus/vote_reservation_test.go b/pkg/consensus/vote_reservation_test.go new file mode 100644 index 000000000..f4e5c418d --- /dev/null +++ b/pkg/consensus/vote_reservation_test.go @@ -0,0 +1,330 @@ +package consensus + +import ( + "crypto/ed25519" + "errors" + "math" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func reservedTestConfig(dir string) VotingConfig { + return VotingConfig{Identity: voterTestKey(11), AuthorizedVoter: voterTestKey(12), VoteAccount: solana.PublicKey(voterTestKey(13).Public().(ed25519.PublicKey)), HistoryDir: dir, Genesis: solana.Hash{1}, ReservedHistory: true, InitializeVoteReservation: true, WaitToVoteSlot: 40, ReadyToVote: func(uint64) bool { return true }, EpochForSlot: func(uint64) uint64 { return 7 }, Peers: func([]alpenglow.ValidatorStake) []alpenglow.VotorPeer { return nil }} +} + +func openReservedTestVoter(t *testing.T, cfg VotingConfig, root uint64) (*alpenglowVoter, error) { + t.Helper() + e, err := NewEngine(Config{AlpenglowIdentity: cfg.Identity, AlpenglowShredVersion: 0x1234}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, e.Close()) }) + set := voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount) + e.SetAlpenglowEpochLookup(cfg.EpochForSlot) + require.NoError(t, e.SetAlpenglowValidatorSet(set)) + block := alpenglow.BlockID{Slot: root, Hash: solana.Hash{byte(root)}} + e.SetAlpenglowRoot(block) + v, err := newAlpenglowVoterUnstarted(e, cfg, block, []alpenglow.ValidatorSet{set}) + if err == nil { + t.Cleanup(func() { require.NoError(t, v.close()) }) + } + return v, err +} + +func reserveThrough(t *testing.T, r *signingReservation, slot uint64) { + t.Helper() + r.request(slot) + require.Eventually(t, func() bool { return r.through.Load() >= slot }, time.Second, time.Millisecond) +} + +// Simulate loss of this process without executing the clean shutdown protocol. +func crashReservedTestVoter(t *testing.T, v *alpenglowVoter) { + t.Helper() + v.shutdownOnce.Do(func() { + v.closeOnce.Do(func() { close(v.done) }) + v.wg.Wait() + v.reservation.halt() + if v.historyWriter != nil { + require.NoError(t, v.historyWriter.close()) + } + require.NoError(t, v.broadcaster.Close()) + require.NoError(t, v.historyLock.Close()) + }) +} + +func TestReservedVotingLostHistorySuffixAndRepeatedCrash(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 60) + baseline, err := os.ReadFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node)) + require.NoError(t, err) + voted, err := v.cast(alpenglow.NewNotarizationVote(60, solana.Hash{1}), false) + require.NoError(t, err) + require.True(t, voted) + oldH := v.reservation.through.Load() + crashReservedTestVoter(t, v) + // Reproduce a host crash retaining a valid older version of detailed history. + require.NoError(t, os.WriteFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node), baseline, 0600)) + cfg.InitializeVoteReservation = false + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Equal(t, oldH, resumed.reservation.recoverThrough) + for _, vote := range []alpenglow.Vote{alpenglow.NewNotarizationVote(60, solana.Hash{2}), alpenglow.NewSkipVote(60), alpenglow.NewFinalizationVote(60), alpenglow.NewNotarizationFallbackVote(60, solana.Hash{2}), alpenglow.NewSkipFallbackVote(60), alpenglow.NewSkipVote(oldH + 1)} { + // Check at the signing boundary, including the restoration bypass. + for _, normal := range []bool{false, true} { + _, _, err := resumed.sign(vote, normal) + require.ErrorIs(t, err, errVoterNotReady) + } + } + require.Equal(t, oldH, resumed.reservation.through.Load(), "recovery must not keep moving its target") + crashReservedTestVoter(t, resumed) + resumed, err = openReservedTestVoter(t, cfg, oldH) + require.NoError(t, err) + require.Equal(t, oldH, resumed.reservation.recoverThrough) + reserveThrough(t, resumed.reservation, oldH+1) + voted, err = resumed.cast(alpenglow.NewSkipVote(oldH+1), false) + require.NoError(t, err) + require.True(t, voted) + voted, err = resumed.cast(alpenglow.NewSkipVote(oldH), false) + require.NoError(t, err) + require.False(t, voted) +} + +func TestReservedVotingCleanMarkerConsumedBeforeSigning(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + voted, err := v.cast(alpenglow.NewSkipVote(44), false) + require.NoError(t, err) + require.True(t, voted) + require.NoError(t, v.close()) + r, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.NotEmpty(t, r.CleanHistoryDigest) + cfg.InitializeVoteReservation = false + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Zero(t, resumed.reservation.recoverThrough) + require.True(t, resumed.history.HasSkipped(44)) + r, err = alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) + voted, err = resumed.cast(alpenglow.NewSkipVote(45), false) + require.NoError(t, err) + require.True(t, voted) + require.False(t, resumed.reservation.allow(45, 39, true), "clean vote history does not authorize repeating leader blocks") + crashReservedTestVoter(t, resumed) + again, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Equal(t, r.Through, again.reservation.recoverThrough) + // A shutdown before recovering the uncertain range must not mark it clean. + require.NoError(t, again.close()) + r, err = alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Empty(t, r.CleanHistoryDigest) +} + +func TestReservedVotingRejectsMissingCorruptOrWrongDomain(t *testing.T) { + for _, which := range []string{"missing_history", "missing_bound", "corrupt_bound", "genesis", "authorized", "vote_account", "synchronous"} { + t.Run(which, func(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + crashReservedTestVoter(t, v) + cfg.InitializeVoteReservation = false + switch which { + case "missing_history": + require.NoError(t, os.Remove(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node))) + case "missing_bound": + require.NoError(t, os.Remove(alpenglow.VoteReservationFilename(cfg.HistoryDir, v.node))) + cfg.InitializeVoteReservation = true + case "corrupt_bound": + require.NoError(t, os.WriteFile(alpenglow.VoteReservationFilename(cfg.HistoryDir, v.node), []byte("{"), 0600)) + case "genesis": + cfg.Genesis = solana.Hash{2} + case "authorized": + cfg.AuthorizedVoter = voterTestKey(19) + case "vote_account": + cfg.VoteAccount = solana.PublicKey{9} + case "synchronous": + cfg.ReservedHistory = false + } + _, err = openReservedTestVoter(t, cfg, 39) + require.Error(t, err) + }) + } +} + +func TestReservedVotingRequiresEnrollmentAndExclusiveOwner(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + cfg.InitializeVoteReservation = false + _, err := openReservedTestVoter(t, cfg, 39) + require.Error(t, err) + cfg.InitializeVoteReservation = true + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + _, err = openReservedTestVoter(t, cfg, 39) + require.ErrorContains(t, err, "already owned") + require.NoError(t, v.close()) +} + +func TestSigningReservationUnacknowledgedSyncCannotAuthorize(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + r := &signingReservation{record: alpenglow.VoteReservation{Through: 64, Generation: 1}, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + r.through.Store(64) + var calls atomic.Uint64 + r.persist = func(next alpenglow.VoteReservation) error { + calls.Add(1) + close(entered) + <-release + return nil + } + go r.run() + require.True(t, r.allow(64, 64, false)) + <-entered + require.Equal(t, uint64(64), r.through.Load()) + require.False(t, r.allow(65, 64, false)) + close(release) + require.Eventually(t, func() bool { return r.through.Load() > 64 }, time.Second, time.Millisecond) + require.True(t, r.allow(65, 64, false)) + r.halt() + require.Equal(t, uint64(1), calls.Load()) +} + +func TestSigningReservationUncertainWriteSurvivesRestart(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + // Stop the original worker, then exercise a fresh worker against the real record. + v.reservation.halt() + record, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + r := &signingReservation{record: record, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{})} + r.through.Store(record.Through) + wrote := make(chan struct{}, 1) + r.persist = func(next alpenglow.VoteReservation) error { + err := alpenglow.SaveVoteReservation(cfg.HistoryDir, next, cfg.Identity) + select { + case wrote <- struct{}{}: + default: + } + if err != nil { + return err + } + return errors.New("injected lost sync acknowledgement") + } + v.reservation = r + go r.run() + require.False(t, r.allow(60, 39, false)) + <-wrote + r.halt() + require.Equal(t, record.Through, r.through.Load()) + durable, err := alpenglow.LoadVoteReservation(cfg.HistoryDir, v.node) + require.NoError(t, err) + require.Greater(t, durable.Through, record.Through) + require.ErrorContains(t, r.seal(cfg.HistoryDir, v.history, cfg.Identity), "uncertain") + crashReservedTestVoter(t, v) + cfg.InitializeVoteReservation = false + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.Equal(t, durable.Through, resumed.reservation.recoverThrough) +} + +func TestSigningReservationNeverWraps(t *testing.T) { + r := &signingReservation{record: alpenglow.VoteReservation{Through: 64, Generation: 1}, wake: make(chan struct{}, 1), changed: make(chan struct{}, 1), stop: make(chan struct{}), done: make(chan struct{}), persist: func(alpenglow.VoteReservation) error { t.Error("overflow attempted persistence"); return nil }} + r.through.Store(64) + go r.run() + require.False(t, r.allow(math.MaxUint64, 64, false)) + r.halt() + require.Equal(t, uint64(64), r.through.Load()) +} + +func TestReservationRetryRechecksFinality(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + event := voterEvent{kind: voterEventBlockTimeout, slot: 44} + require.NoError(t, v.handle(event)) + require.NotEmpty(t, v.reservationEvents) + reserveThrough(t, v.reservation, 44) + v.engine.SetAlpenglowRoot(alpenglow.BlockID{Slot: 47, Hash: solana.Hash{47}}) + pending := v.reservationEvents + v.reservationEvents = nil + for _, e := range pending { + require.NoError(t, v.handle(e)) + } + require.False(t, v.history.HasSkipped(44), "finalized work must not be signed after a delayed ack") +} + +func TestReservedVotingEverySignatureTypeAtBound(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + h := v.reservation.through.Load() + // Freeze acknowledgement while leaving the signing guard active. + v.reservation.halt() + v.reservation.stopped.Store(false) + defer v.reservation.stopped.Store(true) + for _, slot := range []uint64{h, h + 1} { + votes := []alpenglow.Vote{alpenglow.NewNotarizationVote(slot, solana.Hash{2}), alpenglow.NewSkipVote(slot), alpenglow.NewFinalizationVote(slot), alpenglow.NewNotarizationFallbackVote(slot, solana.Hash{2}), alpenglow.NewSkipFallbackVote(slot)} + for _, vote := range votes { + for _, normal := range []bool{false, true} { + _, _, err := v.sign(vote, normal) + if slot == h { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, errVoterNotReady) + } + } + } + } +} + +func TestReservedCleanDigestMismatchUsesCrashRecovery(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + baseline, err := os.ReadFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node)) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + voted, err := v.cast(alpenglow.NewSkipVote(44), false) + require.NoError(t, err) + require.True(t, voted) + require.NoError(t, v.close()) + require.NoError(t, os.WriteFile(alpenglow.VoteHistoryFilename(cfg.HistoryDir, v.node), baseline, 0600)) + resumed, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + require.NotZero(t, resumed.reservation.recoverThrough) +} + +func TestReservationLoopRetriesAfterAcknowledgement(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + v.start() + require.NoError(t, v.enqueue(voterEvent{kind: voterEventBlockTimeout, slot: 44})) + // Read via stats, not mutable voter history, while its loop is running. + require.Eventually(t, func() bool { v.landingMu.RLock(); defer v.landingMu.RUnlock(); return v.stats.VotesCastThisRun == 4 }, time.Second, time.Millisecond) + require.NoError(t, v.close()) + for slot := uint64(44); slot <= 47; slot++ { + require.True(t, v.history.HasSkipped(slot)) + } +} + +func TestReservationRetainsWindowCrossingBound(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 44) + h := v.reservation.through.Load() // 76: window ends at 79. + v.retainReservationEvent(voterEvent{kind: voterEventBlockTimeout, slot: h}) + require.Len(t, v.reservationEvents, 1, "trailing skip slots require retry even if the first slot fits") +} diff --git a/pkg/consensus/voter.go b/pkg/consensus/voter.go index e5a220d96..4a9cbbc15 100644 --- a/pkg/consensus/voter.go +++ b/pkg/consensus/voter.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "math" + "os" "sort" "sync" "time" @@ -38,21 +39,30 @@ type VotingPeerSource func(validators []alpenglow.ValidatorStake) []alpenglow.Vo // account address; AuthorizedVoter is the Ed25519 signer from which its BLS key // was registered. type VotingConfig struct { - Identity ed25519.PrivateKey - AuthorizedVoter ed25519.PrivateKey - VoteAccount solana.PublicKey - HistoryDir string - EpochForSlot func(slot uint64) uint64 - Peers VotingPeerSource - SlotDuration time.Duration - WaitToVoteSlot uint64 - ReadyToVote func(slot uint64) bool + Identity ed25519.PrivateKey + AuthorizedVoter ed25519.PrivateKey + VoteAccount solana.PublicKey + HistoryDir string + ReservedHistory bool + InitializeVoteReservation bool + Genesis solana.Hash + EpochForSlot func(slot uint64) uint64 + Peers VotingPeerSource + SlotDuration time.Duration + WaitToVoteSlot uint64 // Inclusive minimum for new votes; authenticated history restoration is separate + ReadyToVote func(slot uint64) bool } // VotingStats exposes positive network evidence separately from local casting. // NetworkLandedVotes counts unique persisted votes whose rank appeared in the // exact BLS-verified certificate proof received over Votor QUIC. type VotingStats struct { + HistorySnapshotsSubmitted uint64 `json:"history_snapshots_submitted,omitempty"` + HistorySnapshotsWritten uint64 `json:"history_snapshots_written,omitempty"` + HistorySnapshotsCoalesced uint64 `json:"history_snapshots_coalesced,omitempty"` + ReservedHistory bool `json:"reserved_history"` + SigningReservedThrough uint64 `json:"signing_reserved_through,omitempty"` + RecoveryThrough uint64 `json:"recovery_through,omitempty"` Enabled bool `json:"enabled"` VotesCastThisRun uint64 `json:"votes_cast_this_run"` NetworkLandedVotes uint64 `json:"network_landed_votes"` @@ -68,7 +78,7 @@ type VotingStats struct { BroadcastPeerQueueDrops uint64 `json:"broadcast_peer_queue_drops"` BroadcastPeerQueueDiscarded uint64 `json:"broadcast_peer_queue_discarded"` BroadcastPeerSendTimeouts uint64 `json:"broadcast_peer_send_timeouts"` - BroadcastPeerQueueMaxDelay time.Duration `json:"broadcast_peer_queue_max_delay"` + BroadcastPeerQueueMaxDelay time.Duration `json:"broadcast_peer_queue_max_delay_ns"` BroadcastPeerQueues []alpenglow.VotorPeerQueueStats `json:"broadcast_peer_queues,omitempty"` BroadcastDesiredPeers int `json:"broadcast_desired_peers"` BroadcastActiveConnections int `json:"broadcast_active_connections"` @@ -93,6 +103,7 @@ const ( voterEventValidatorSet voterEventRoot voterEventNetworkCertificate + voterEventDurableRoot ) type voterEvent struct { @@ -114,43 +125,49 @@ type pendingVotorBlock struct { // history decisions run on loop; validator-set snapshots are protected only so // the outbound peer callback can read them from broadcast workers. type alpenglowVoter struct { - engine *AlpenglowObserverEngine - identity ed25519.PrivateKey - node solana.PublicKey - voteAccount solana.PublicKey - signer *alpenglow.BLSSigner - historyDir string - history *alpenglow.VoteHistory - epochForSlot func(uint64) uint64 - peerSource VotingPeerSource - slotDuration time.Duration - waitToVoteSlot uint64 - readyToVote func(slot uint64) bool - broadcaster *alpenglow.VotorBroadcaster - events chan voterEvent - done chan struct{} - startOnce sync.Once - closeOnce sync.Once - wg sync.WaitGroup - setsMu sync.RWMutex - sets map[uint64]alpenglow.ValidatorSet - restored map[alpenglow.VoteMessageKey]bool - pending map[uint64][]pendingVotorBlock - receivedShred map[uint64]bool - timeoutsSet map[uint64]bool - executedBlocks map[alpenglow.BlockID]bool - highestFinal uint64 - lastFinalizedAt time.Time - votingStarted bool - latestLiveSlot uint64 - standstillSlot *uint64 - refreshQueue []alpenglow.Message - refreshCursor int - lastWarn map[uint64]time.Time - landingMu sync.RWMutex - landed map[alpenglow.VoteMessageKey]struct{} - stats VotingStats - lastStatsLog time.Time + engine *AlpenglowObserverEngine + identity ed25519.PrivateKey + node solana.PublicKey + voteAccount solana.PublicKey + signer *alpenglow.BLSSigner + historyDir string + historyLock *os.File + reservation *signingReservation + historyWriter *voteHistoryWriter + reservationEvents []voterEvent + shutdownOnce sync.Once + shutdownErr error + history *alpenglow.VoteHistory + epochForSlot func(uint64) uint64 + peerSource VotingPeerSource + slotDuration time.Duration + waitToVoteSlot uint64 + readyToVote func(slot uint64) bool + broadcaster *alpenglow.VotorBroadcaster + events chan voterEvent + done chan struct{} + startOnce sync.Once + closeOnce sync.Once + wg sync.WaitGroup + setsMu sync.RWMutex + sets map[uint64]alpenglow.ValidatorSet + restored map[alpenglow.VoteMessageKey]bool + pending map[uint64][]pendingVotorBlock + receivedShred map[uint64]bool + timeoutsSet map[uint64]bool + executedBlocks map[alpenglow.BlockID]bool + highestFinal uint64 + lastFinalizedAt time.Time + votingStarted bool + latestLiveSlot uint64 + standstillSlot *uint64 + refreshQueue []alpenglow.Message + refreshCursor int + lastWarn map[uint64]time.Time + landingMu sync.RWMutex + landed map[alpenglow.VoteMessageKey]struct{} + stats VotingStats + lastStatsLog time.Time // beforeVoteGuard is a deterministic test seam for invalidation races. It // is nil in production. beforeVoteGuard func(alpenglow.BlockID) @@ -168,6 +185,9 @@ func newAlpenglowVoterUnstarted(engine *AlpenglowObserverEngine, cfg VotingConfi } func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfig, root alpenglow.BlockID, start bool, sets []alpenglow.ValidatorSet) (*alpenglowVoter, error) { + if cfg.InitializeVoteReservation && !cfg.ReservedHistory { + return nil, errors.New("initialize-vote-reservation requires reserved-vote-history") + } if engine == nil { return nil, fmt.Errorf("enable Alpenglow voting: nil consensus engine") } @@ -197,17 +217,52 @@ func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfi if node != engineNode { return nil, fmt.Errorf("enable Alpenglow voting: identity %s does not match consensus transport identity %s", node, engineNode) } + historyLock, err := alpenglow.LockVoteHistory(cfg.HistoryDir, node) + if err != nil { + return nil, err + } + keepLock := false + defer func() { + if !keepLock { + historyLock.Close() + } + }() + if !cfg.ReservedHistory { + if _, err := os.Stat(alpenglow.VoteReservationFilename(cfg.HistoryDir, node)); !errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("existing or unreadable vote reservation requires reserved history mode") + } + } history, err := alpenglow.LoadVoteHistory(cfg.HistoryDir, node) if err != nil { if !errors.Is(err, alpenglow.ErrVoteHistoryNotFound) { return nil, fmt.Errorf("enable Alpenglow voting: refuse unsafe vote-history reset: %w", err) } + if cfg.ReservedHistory { + if _, err := os.Stat(alpenglow.VoteReservationFilename(cfg.HistoryDir, node)); !errors.Is(err, os.ErrNotExist) || !cfg.InitializeVoteReservation { + return nil, fmt.Errorf("missing history for reserved voter; refusing automatic reset") + } + } history = alpenglow.NewVoteHistory(node, root.Slot) if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { return nil, fmt.Errorf("initialize Alpenglow vote history: %w", err) } mlog.Log.FileOnlyf("ALPENGLOW voting: created new vote history for %s at root %d; do not reuse this vote account on another validator", node, root.Slot) } + if history.ReservationRequired && !cfg.ReservedHistory { + return nil, errors.New("reserved vote history cannot be opened in synchronous mode") + } + var reservation *signingReservation + if cfg.ReservedHistory { + reservation, err = openSigningReservation(cfg, node, engine.shredVersion, history) + if err != nil { + return nil, err + } + defer func() { + if !keepLock { + reservation.halt() + } + }() + } if history.Root < root.Slot { history.SetRoot(root.Slot) if err := alpenglow.SaveVoteHistory(cfg.HistoryDir, history, cfg.Identity); err != nil { @@ -225,6 +280,8 @@ func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfi voteAccount: cfg.VoteAccount, signer: signer, historyDir: cfg.HistoryDir, + historyLock: historyLock, + reservation: reservation, history: history, epochForSlot: cfg.EpochForSlot, peerSource: cfg.Peers, @@ -266,6 +323,12 @@ func newAlpenglowVoterWithStart(engine *AlpenglowObserverEngine, cfg VotingConfi return nil, err } v.broadcaster = broadcaster + if reservation != nil { + v.historyWriter = newVoteHistoryWriter(func(snapshot *alpenglow.VoteHistorySnapshot) error { + return alpenglow.SaveReservedVoteHistorySnapshot(v.historyDir, snapshot) + }, v.failHistoryWrite) + } + keepLock = true if start { v.start() } @@ -305,12 +368,25 @@ func (v *alpenglowVoter) loop() { v.closeOnce.Do(func() { close(v.done) }) v.wg.Done() }() + var reservationChanged <-chan struct{} + if v.reservation != nil { + reservationChanged = v.reservation.changed + } ticker := time.NewTicker(time.Second) defer ticker.Stop() for { select { case <-v.done: return + case <-reservationChanged: + pending := v.reservationEvents + v.reservationEvents = nil + for _, event := range pending { + if err := v.handle(event); err != nil { + v.engine.latchSafetyError(fmt.Errorf("reservation retry: %w", err)) + return + } + } case event := <-v.events: if err := v.handle(event); err != nil { v.engine.latchSafetyError(fmt.Errorf("voting engine: %w", err)) @@ -329,6 +405,11 @@ func (v *alpenglowVoter) loop() { } func (v *alpenglowVoter) handle(event voterEvent) error { + v.retainReservationEvent(event) + return v.handleEvent(event) +} + +func (v *alpenglowVoter) handleEvent(event voterEvent) error { floor := v.admissionFloor() if event.kind == voterEventBlock && v.engine.ensureChain().IsObjectivelyInvalidBlock(event.block.Block) { return nil @@ -368,6 +449,9 @@ func (v *alpenglowVoter) handle(event voterEvent) error { return nil } return v.saveHistory() + case voterEventDurableRoot: + root := v.engine.applyAlpenglowDurableRoot(event.slot) + return v.handleEvent(voterEvent{kind: voterEventRoot, root: root}) case voterEventNetworkCertificate: v.recordNetworkCertificate(event.certificate) return nil @@ -482,9 +566,9 @@ func (v *alpenglowVoter) handleConsensus(event alpenglow.ConsensusEvent) error { func (v *alpenglowVoter) admissionFloor() uint64 { floor := v.history.Root - if v.highestFinal > floor { - floor = v.highestFinal - } + // highestFinal tracks network progress and standstill, not retirement of + // our own decisions. Keep ParentReady, pending replay and exact vote history + // available until the retained pool or an ordered durable root retires them. if engineFloor := v.engine.alpenglowVoteActionFloor(); engineFloor > floor { floor = engineFloor } @@ -666,7 +750,7 @@ func (v *alpenglowVoter) castTarget(vote alpenglow.Vote, restoring bool, guarded if !restoring && v.beforeVoteGuard != nil { v.beforeVoteGuard(guardedBlock) } - // Hold through signing, durable history, pool admission, and broadcast. + // Hold through signing, history recording, pool admission, and broadcast. // Objective invalidation takes the write side before changing the chain, // so a new or restored vote is wholly before it or sees the tombstone. v.engine.invalidActionMu.RLock() @@ -687,18 +771,22 @@ func (v *alpenglowVoter) castTarget(vote alpenglow.Vote, restoring bool, guarded return false, nil } if !restoring { - // Finality can advance while the BLS signature is computed. Avoid a - // durable stale record when that race is already visible here; if it - // advances later, atomic admission below classifies it benignly. + // Retention/root pruning can advance while the BLS signature is computed. + // Avoid an expired record if that race is already visible here; atomic + // admission below classifies a later pruning race benignly. if vote.Slot <= v.admissionFloor() { return false, nil } if err := v.history.AddVote(vote); err != nil { return false, fmt.Errorf("record %s vote at slot %d: %w", vote.Type, vote.Slot, err) } - // Pool admission may synchronously assemble and publish a certificate. - // Persist the anti-equivocation record first so no externally visible - // proof can survive a crash without its signed local history. + // Pool admission can publish a certificate. In reserved mode the durable + // upper bound covers loss of this unsynchronized history replacement; + // synchronous mode still persists the exact history before admission. + // sign computed BLS bytes in RAM, but nothing may expose them before + // this boundary succeeds. In reserved mode saveHistory only queues the + // snapshot: restart safety comes from the durable reservation checked + // before sign, not from assuming this snapshot reached durable storage. if err := v.saveHistory(); err != nil { return false, err } @@ -732,7 +820,16 @@ func (v *alpenglowVoter) castTarget(vote alpenglow.Vote, restoring bool, guarded return true, nil } +// sign checks reservation recovery even when restoration bypasses the live +// joining gate. Re-signing a saved vote is still signing; the presence of an +// older valid history file cannot prove that its lost suffix was conflict-free. func (v *alpenglowVoter) sign(vote alpenglow.Vote, respectVotingGate bool) (alpenglow.VoteMessage, alpenglow.VoteVerifyResult, error) { + if err := v.engine.safetyError(); err != nil { + return alpenglow.VoteMessage{}, alpenglow.VoteVerifyResult{}, err + } + if v.reservation != nil && !v.reservation.allow(vote.Slot, v.engine.alpenglowVerifiedFinalityFloor(), false) { + return alpenglow.VoteMessage{}, alpenglow.VoteVerifyResult{}, fmt.Errorf("%w: waiting for verified recovery or durable signing reservation", errVoterNotReady) + } if respectVotingGate { if err := v.votingGateError(vote.Slot); err != nil { return alpenglow.VoteMessage{}, alpenglow.VoteVerifyResult{}, err @@ -773,7 +870,7 @@ func (v *alpenglowVoter) votingGateError(slot uint64) error { return fmt.Errorf("%w: slot %d is at or below consensus action floor %d", errVoterNotReady, slot, floor) } if slot < v.waitToVoteSlot { - return fmt.Errorf("%w: waiting for startup watermark slot %d", errVoterNotReady, v.waitToVoteSlot) + return fmt.Errorf("%w: waiting for voting cutoff slot %d", errVoterNotReady, v.waitToVoteSlot) } // ReadyToVote is a startup join guard, not a perpetual clock check. Once an // accepted live block or vote joins Votor, verified ParentReady and timeout @@ -782,6 +879,9 @@ func (v *alpenglowVoter) votingGateError(slot uint64) error { if !v.votingStarted && v.readyToVote != nil && !v.readyToVote(slot) { return fmt.Errorf("%w: slot is still behind the startup live voting window", errVoterNotReady) } + if v.reservation != nil && !v.reservation.allow(slot, v.engine.alpenglowVerifiedFinalityFloor(), false) { + return fmt.Errorf("%w: waiting for verified recovery or durable signing reservation", errVoterNotReady) + } return nil } @@ -842,6 +942,9 @@ func (v *alpenglowVoter) votorTransportValidators() []alpenglow.ValidatorStake { } func (v *alpenglowVoter) restoreVotesForEpoch(epoch uint64) error { + if v.reservation != nil && v.reservation.recoverThrough != 0 { + return nil + } floor := v.admissionFloor() for _, vote := range v.history.VotesAfter(v.history.Root - minU64(v.history.Root, 1)) { // Keep every signed history entry for anti-equivocation, but do not @@ -889,6 +992,9 @@ func (v *alpenglowVoter) restoreVotesForEpoch(epoch uint64) error { // persist-before-admission crash window without trusting process-local invalid // block state across a restart. func (v *alpenglowVoter) restoreVotesForBlock(block alpenglow.BlockID) (bool, error) { + if v.reservation != nil && v.reservation.recoverThrough != 0 { + return false, nil + } if block.Slot <= v.admissionFloor() { return false, nil } @@ -1091,13 +1197,29 @@ func (v *alpenglowVoter) isIdentityStaked(slot uint64) bool { return false } +// saveHistory is a publication boundary with different persistence semantics: +// synchronous mode acknowledges durable exact history; reserved mode acknowledges +// an immutable queued snapshot only. The latter relies on the signing reservation. func (v *alpenglowVoter) saveHistory() error { + if v.historyWriter != nil { + snapshot, err := alpenglow.PrepareReservedVoteHistory(v.history, v.identity) + if err != nil { + return err + } + return v.historyWriter.submit(snapshot) + } if err := alpenglow.SaveVoteHistory(v.historyDir, v.history, v.identity); err != nil { return fmt.Errorf("persist vote history before consensus publication: %w", err) } return nil } +func (v *alpenglowVoter) failHistoryWrite(err error) { + v.engine.latchSafetyError(err) + mlog.Log.Errorf("ALPENGLOW VOTING SAFETY: %v", err) + v.closeOnce.Do(func() { close(v.done) }) +} + func (v *alpenglowVoter) recordNetworkCertificate(cert alpenglow.Certificate) { set, ok := v.validatorSet(cert.Slot) if !ok { @@ -1206,6 +1328,14 @@ func (v *alpenglowVoter) snapshot() VotingStats { stats := v.stats v.landingMu.RUnlock() stats.Enabled = true + if v.historyWriter != nil { + stats.HistorySnapshotsSubmitted, stats.HistorySnapshotsWritten, stats.HistorySnapshotsCoalesced = v.historyWriter.counters() + } + if v.reservation != nil { + stats.ReservedHistory = true + stats.SigningReservedThrough = v.reservation.through.Load() + stats.RecoveryThrough = v.reservation.recoverThrough + } if v.broadcaster != nil { broadcast := v.broadcaster.Stats() stats.BroadcastMessagesQueued = broadcast.MessagesQueued @@ -1238,7 +1368,7 @@ func (v *alpenglowVoter) maybeLogStats() { } v.lastStatsLog = time.Now() stats := v.snapshot() - mlog.Log.FileOnlyf("alpenglow voting stats: votes_cast_this_run=%d network_landed=%d last_landed_slot=%d broadcast_queued=%d broadcast_dropped=%d peer_sends=%d peer_sends_skipped=%d peer_send_errors=%d desired_peers=%d active_connections=%d pending_connections=%d connection_attempts=%d connection_errors=%d connection_jobs_dropped=%d", + mlog.Log.FileOnlyf("alpenglow voting stats: votes_cast_this_run=%d network_landed=%d last_landed_slot=%d broadcast_queued=%d broadcast_dropped=%d peer_sends=%d peer_sends_skipped=%d peer_send_errors=%d peer_queue_drops=%d peer_queue_discarded=%d peer_send_timeouts=%d peer_queue_max_delay=%s desired_peers=%d active_connections=%d pending_connections=%d connection_attempts=%d connection_errors=%d connection_jobs_dropped=%d reserved_history=%t signing_through=%d recovery_through=%d history_submitted=%d history_written=%d history_coalesced=%d", stats.VotesCastThisRun, stats.NetworkLandedVotes, stats.LastNetworkLandedSlot, @@ -1247,12 +1377,18 @@ func (v *alpenglowVoter) maybeLogStats() { stats.BroadcastPeerSends, stats.BroadcastPeerSendsSkipped, stats.BroadcastPeerSendErrors, + stats.BroadcastPeerQueueDrops, + stats.BroadcastPeerQueueDiscarded, + stats.BroadcastPeerSendTimeouts, + stats.BroadcastPeerQueueMaxDelay, stats.BroadcastDesiredPeers, stats.BroadcastActiveConnections, stats.BroadcastPendingConnections, stats.BroadcastConnectionAttempts, stats.BroadcastConnectionErrors, stats.BroadcastConnectionJobsDropped, + stats.ReservedHistory, stats.SigningReservedThrough, stats.RecoveryThrough, + stats.HistorySnapshotsSubmitted, stats.HistorySnapshotsWritten, stats.HistorySnapshotsCoalesced, ) } @@ -1284,9 +1420,28 @@ func (v *alpenglowVoter) close() error { if v == nil { return nil } - v.closeOnce.Do(func() { close(v.done) }) - v.wg.Wait() - return v.broadcaster.Close() + v.shutdownOnce.Do(func() { + v.closeOnce.Do(func() { close(v.done) }) + v.wg.Wait() + if v.reservation != nil { + v.reservation.halt() + if v.historyWriter != nil { + v.shutdownErr = v.historyWriter.close() + } + // Exiting normally is insufficient: an unresolved recovery barrier, + // writer failure or safety fault must leave the session unsealed. + floor := v.engine.alpenglowVerifiedFinalityFloor() + if v.shutdownErr == nil && v.engine.safetyError() == nil && floor >= v.reservation.recoverThrough { + v.history.SetRoot(floor) + v.shutdownErr = v.reservation.seal(v.historyDir, v.history, v.identity) + if v.shutdownErr == nil { + mlog.Log.Infof("ALPENGLOW signing reservation: clean history sealed at root=%d through=%d", v.history.Root, v.reservation.through.Load()) + } + } + } + v.shutdownErr = errors.Join(v.shutdownErr, v.broadcaster.Close(), v.historyLock.Close()) + }) + return v.shutdownErr } func pendingContains(blocks []pendingVotorBlock, candidate pendingVotorBlock) bool { diff --git a/pkg/consensus/voter_finality_ordering_test.go b/pkg/consensus/voter_finality_ordering_test.go new file mode 100644 index 000000000..a53abb640 --- /dev/null +++ b/pkg/consensus/voter_finality_ordering_test.go @@ -0,0 +1,192 @@ +package consensus + +import ( + "context" + "crypto/ed25519" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Keep the actor unstarted so tests can place finality, replay and durable +// promotion in a deterministic order, using the real engine event queue. +func newOrderingTestVoter(t *testing.T, reserved bool) (*AlpenglowObserverEngine, *alpenglowVoter) { + t.Helper() + cfg := reservedTestConfig(t.TempDir()) + cfg.ReservedHistory = reserved + cfg.InitializeVoteReservation = reserved + root := alpenglow.BlockID{Slot: 39, Hash: solana.Hash{39}} + e, err := NewEngine(Config{AlpenglowIdentity: cfg.Identity, AlpenglowShredVersion: 0x1234}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, e.Close()) }) + set := voterTestValidatorSet(t, cfg.Identity, cfg.AuthorizedVoter, cfg.VoteAccount) + e.SetAlpenglowEpochLookup(cfg.EpochForSlot) + require.NoError(t, e.SetAlpenglowValidatorSet(set)) + e.SetAlpenglowRoot(root) + v, err := newAlpenglowVoterUnstarted(e, cfg, root, []alpenglow.ValidatorSet{set}) + require.NoError(t, err) + e.voter = v + if reserved { + reserveThrough(t, v.reservation, 44) + } + // Equivalent to startup's trusted-root ParentReady seed. + require.True(t, v.history.AddParentReady(40, root)) + return e, v +} + +func drainOrderingEvents(t *testing.T, v *alpenglowVoter) { + t.Helper() + for i := 0; i < 1000; i++ { + select { + case event := <-v.events: + require.NoError(t, v.handle(event)) + default: + return + } + } + t.Fatal("voter event queue did not drain") +} + +func observeOrderingBlock(t *testing.T, e *AlpenglowObserverEngine, slot uint64) alpenglow.BlockID { + t.Helper() + id := alpenglow.BlockID{Slot: slot, Hash: solana.Hash{byte(slot)}} + require.NoError(t, e.ObserveBlock(context.Background(), BlockObservation{Source: "ordering-test", Block: &block.Block{ + Slot: slot, ParentSlot: slot - 1, + AlpenglowBlockID: [32]byte(id.Hash), HasAlpenglowBlockID: true, + AlpenglowParentBlockID: [32]byte{byte(slot - 1)}, HasAlpenglowParentBlockID: true, + }})) + return id +} + +func finalizeOrderingBlock(t *testing.T, e *AlpenglowObserverEngine, v *alpenglowVoter, id alpenglow.BlockID) { + t.Helper() + // The two peers supply the 60% slow-finality quorum without our vote; + // adding our 30% notarization later can produce a fast certificate. + for _, vote := range []alpenglow.Vote{alpenglow.NewNotarizationVote(id.Slot, id.Hash), alpenglow.NewFinalizationVote(id.Slot)} { + for rank, key := range []ed25519.PrivateKey{voterTestKey(21), voterTestKey(22)} { + peer := signedVerifiedVoterPeerVote(t, e, v.sets[7], key, uint16(rank+1), vote) + _, err := e.acceptVerifiedVoteResult(peer) + require.NoError(t, err) + } + } + require.Equal(t, id.Slot, e.ensureChain().Snapshot().LatestDirectFinalizedBlock.Slot) +} + +func TestAlpenglowVoterReplaysFourBlocksAfterNetworkFinality(t *testing.T) { + for _, reserved := range []bool{false, true} { + name := "synchronous" + if reserved { + name = "reserved" + } + t.Run(name, func(t *testing.T) { + e, v := newOrderingTestVoter(t, reserved) + for slot := uint64(40); slot <= 43; slot++ { + id := observeOrderingBlock(t, e, slot) + finalizeOrderingBlock(t, e, v, id) + drainOrderingEvents(t, v) + require.Equal(t, slot, v.highestFinal) + require.Less(t, v.admissionFloor(), slot) + require.False(t, v.history.VotedAt(slot), "network finality does not prove local execution") + before := v.snapshot() + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: slot, Source: "ordering-test"})) + drainOrderingEvents(t, v) + hash, ok := v.history.NotarizedVote(slot) + require.True(t, ok, "replay must still notarize after slow finality") + require.Equal(t, id.Hash, hash) + require.Greater(t, v.snapshot().BroadcastMessagesQueued, before.BroadcastMessagesQueued) + require.NoError(t, e.AlpenglowSafetyError()) + } + }) + } +} + +func TestAlpenglowVoterNetworkFinalityDuringLocalAdmission(t *testing.T) { + e, v := newOrderingTestVoter(t, false) + id := observeOrderingBlock(t, e, 40) + called := false + v.beforeLocalVoteInject = func(vote alpenglow.Vote) { + if called { + return + } + called = true + require.Equal(t, alpenglow.NewNotarizationVote(id.Slot, id.Hash), vote) + finalizeOrderingBlock(t, e, v, id) + } + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: id.Slot})) + drainOrderingEvents(t, v) + require.True(t, called) + require.Positive(t, v.snapshot().VotesCastThisRun) + message, _, err := v.sign(alpenglow.NewNotarizationVote(id.Slot, id.Hash), false) + require.NoError(t, err) + require.True(t, e.ensurePool().HasVerifiedVote(message)) + require.NoError(t, e.AlpenglowSafetyError()) +} + +func TestAlpenglowVoterDurableRootCannotOvertakeQueuedReplay(t *testing.T) { + e, v := newOrderingTestVoter(t, true) + id := observeOrderingBlock(t, e, 40) + finalizeOrderingBlock(t, e, v, id) + drainOrderingEvents(t, v) + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: id.Slot})) + e.PruneAlpenglowBefore(id.Slot) + require.Equal(t, uint64(39), e.ensurePool().Snapshot().RootSlot) + require.Contains(t, e.executedReplayBlocks, id, "queued replay must retain its execution proof") + queuedBefore := v.snapshot().BroadcastMessagesQueued + drainOrderingEvents(t, v) + require.Greater(t, v.snapshot().BroadcastMessagesQueued, queuedBefore) + require.Equal(t, id.Slot, v.history.Root) + require.Equal(t, id.Slot, e.ensurePool().Snapshot().RootSlot) + require.NotContains(t, e.executedReplayBlocks, id) + _, ok := v.history.NotarizedVote(id.Slot) + require.True(t, ok, "root vote must remain available to its intra-window child") + child := observeOrderingBlock(t, e, 41) + finalizeOrderingBlock(t, e, v, child) + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: child.Slot})) + drainOrderingEvents(t, v) + hash, ok := v.history.NotarizedVote(child.Slot) + require.True(t, ok) + require.Equal(t, child.Hash, hash) + require.NoError(t, e.AlpenglowSafetyError()) +} + +func TestAlpenglowVoterFinalityPreservesEarlierSkipDecision(t *testing.T) { + e, v := newOrderingTestVoter(t, true) + require.NoError(t, v.history.AddVote(alpenglow.NewSkipVote(40))) + id := observeOrderingBlock(t, e, 40) + finalizeOrderingBlock(t, e, v, id) + drainOrderingEvents(t, v) + require.NoError(t, e.OnReplayResult(context.Background(), SlotReplayResult{Slot: 40})) + drainOrderingEvents(t, v) + require.True(t, v.history.HasSkipped(40)) + _, ok := v.history.NotarizedVote(40) + require.False(t, ok, "late replay must never replace an earlier round-one decision") + require.NoError(t, e.AlpenglowSafetyError()) +} + +func TestReservedRecoveryUsesVerifiedFinalityNotLiveAdmissionFloor(t *testing.T) { + cfg := reservedTestConfig(t.TempDir()) + v, err := openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + reserveThrough(t, v.reservation, 40) + h := v.reservation.through.Load() + crashReservedTestVoter(t, v) + cfg.InitializeVoteReservation = false + v, err = openReservedTestVoter(t, cfg, 39) + require.NoError(t, err) + _, _, err = v.sign(alpenglow.NewSkipVote(h+1), false) + require.ErrorIs(t, err, errVoterNotReady) + id := observeOrderingBlock(t, v.engine, h) + finalizeOrderingBlock(t, v.engine, v, id) + require.Less(t, v.admissionFloor(), h) + require.Equal(t, h, v.engine.alpenglowVerifiedFinalityFloor()) + reserveThrough(t, v.reservation, h+1) + for _, normal := range []bool{false, true} { + _, _, err = v.sign(alpenglow.NewSkipVote(h), normal) + require.ErrorIs(t, err, errVoterNotReady) + _, _, err = v.sign(alpenglow.NewSkipVote(h+1), normal) + require.NoError(t, err) + } +} diff --git a/pkg/consensus/voter_wait_slot_test.go b/pkg/consensus/voter_wait_slot_test.go new file mode 100644 index 000000000..b06a35819 --- /dev/null +++ b/pkg/consensus/voter_wait_slot_test.go @@ -0,0 +1,91 @@ +package consensus + +import ( + "crypto/ed25519" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func waitSlotTestVoter(t *testing.T, cutoff uint64) *alpenglowVoter { + t.Helper() + identity, authorized := voterTestKey(11), voterTestKey(12) + voteAccount := solana.PublicKey(voterTestKey(13).Public().(ed25519.PublicKey)) + set := voterTestValidatorSet(t, identity, authorized, voteAccount) + root := alpenglow.BlockID{Slot: 39, Hash: solana.Hash{0x39}} + engine, err := NewEngine(Config{AlpenglowShredVersion: 0x1234, AlpenglowIdentity: identity}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, engine.Close()) }) + engine.SetAlpenglowEpochLookup(func(uint64) uint64 { return set.Epoch }) + require.NoError(t, engine.SetAlpenglowValidatorSet(set)) + engine.SetAlpenglowRoot(root) + voter, err := newAlpenglowVoterUnstarted(engine, VotingConfig{ + Identity: identity, AuthorizedVoter: authorized, VoteAccount: voteAccount, + HistoryDir: t.TempDir(), EpochForSlot: func(uint64) uint64 { return set.Epoch }, + Peers: func([]alpenglow.ValidatorStake) []alpenglow.VotorPeer { return nil }, + WaitToVoteSlot: cutoff, ReadyToVote: func(uint64) bool { return true }, + }, root, []alpenglow.ValidatorSet{set}) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, voter.close()) }) + return voter +} + +func TestWaitToVoteSlotGatesEveryNewVoteType(t *testing.T) { + voter := waitSlotTestVoter(t, 44) + constructors := []func(uint64) alpenglow.Vote{ + func(slot uint64) alpenglow.Vote { return alpenglow.NewNotarizationVote(slot, solana.Hash{1}) }, + alpenglow.NewFinalizationVote, + alpenglow.NewSkipVote, + func(slot uint64) alpenglow.Vote { return alpenglow.NewNotarizationFallbackVote(slot, solana.Hash{1}) }, + alpenglow.NewSkipFallbackVote, + } + for _, started := range []bool{false, true} { + voter.votingStarted = started + for _, voteAt := range constructors { + vote := voteAt(43) + _, _, err := voter.sign(vote, true) + require.ErrorIs(t, err, errVoterNotReady, "%s started=%t", vote.Type, started) + for _, slot := range []uint64{44, 45} { + message, _, err := voter.sign(voteAt(slot), true) + require.NoError(t, err, "%s slot=%d started=%t", vote.Type, slot, started) + require.Equal(t, voteAt(slot), message.Vote) + } + } + } +} + +func TestWaitToVoteSlotSplitsSkipWindowAndPersistsOnlyAllowedVotes(t *testing.T) { + voter := waitSlotTestVoter(t, 42) + require.NoError(t, voter.trySkipWindow(40)) + for _, slot := range []uint64{40, 41} { + require.False(t, voter.history.VotedAt(slot)) + } + for _, slot := range []uint64{42, 43} { + require.True(t, voter.history.HasSkipped(slot)) + } + restored, err := alpenglow.LoadVoteHistory(voter.historyDir, voter.node) + require.NoError(t, err) + require.Equal(t, voter.history.VotesCast, restored.VotesCast) + require.EqualValues(t, 2, voter.engine.ensurePool().Snapshot().VerifiedVotes) + // Joining live voting must not make older slots eligible afterward. + require.True(t, voter.votingStarted) + voted, err := voter.cast(alpenglow.NewSkipVote(41), false) + require.NoError(t, err) + require.False(t, voted) + require.False(t, voter.history.VotedAt(41)) +} + +func TestWaitToVoteSlotPreservesAuthenticatedHistoryRestoration(t *testing.T) { + voter := waitSlotTestVoter(t, 44) + require.NoError(t, voter.history.AddVote(alpenglow.NewSkipVote(40))) + require.NoError(t, voter.saveHistory()) + restored, err := alpenglow.LoadVoteHistory(voter.historyDir, voter.node) + require.NoError(t, err) + voter.history = restored + require.NoError(t, voter.restoreVotesForEpoch(voter.epochForSlot(40))) + require.EqualValues(t, 1, voter.engine.ensurePool().Snapshot().VerifiedVotes) + require.False(t, voter.votingStarted, "restoring a recorded vote must not bypass startup readiness") + require.ErrorIs(t, voter.votingGateError(41), errVoterNotReady) +}