leader: reduce packing overhead and test near-limit blocks

cmd/mithril/node/node.go

@@ -74,35 +74,37 @@ var (
 		},
 	}
 
-	bootstrapMode                   string // "auto", "snapshot", "new-snapshot", "new-incremental", or "accountsdb"
-	snapshotArchivePath             string
-	incrementalSnapshotFilename     string
-	accountsPath                    string
-	scratchDirectory                string
-	rpcEndpoints                    []string
-	cluster                         string // "alpenglow", "mainnet-beta", "testnet", or "devnet"
-	legacyGenesisHash               string // explicit lineage for pre-binding AccountsDB/ledger artifacts
-	blockSource                     string // "turbine", "rpc", or "lightbringer"
-	lightbringerEndpoint            string
-	repairCatchupMaxGapSlots        int    // Resume gaps up to this fill via turbine repair instead of RPC (0 = off)
-	repairMaxRequestsPerSecond      int    // Repair request-rate ceiling override (0 = adaptive default)
-	blockRPCFallback                bool   // Allow RPC block fetch when > repairCatchupMaxGapSlots behind (default false: shreds only)
-	blockMaxRPS                     int    // Rate limit for block fetching
-	blockMaxInflight                int    // Max concurrent block fetch workers
-	blockTipPollIntervalMs          int    // Tip poll interval in milliseconds
-	blockTipSafetyMargin            int    // Don't fetch within N slots of tip
-	consensusModeFlag               string // raw --consensus-mode value (cobra binding)
-	consensusMode                   string // resolved: "verifying" (default) or "validator"
-	alpenglowObserverBindAddr       string
-	alpenglowMaxMessageBytes        int64
-	alpenglowBLSDST                 string
-	validatorIdentityKeypair        string
-	validatorVoteAccountKeypair     string
-	validatorAuthorizedVoterKeypair string
-	validatorWithdrawerKeypair      string
-	validatorTPUQUICBind            string
-	validatorAdvertisedIP           string
-	validatorSigverifyWorkers       int
+	bootstrapMode                    string // "auto", "snapshot", "new-snapshot", "new-incremental", or "accountsdb"
+	snapshotArchivePath              string
+	incrementalSnapshotFilename      string
+	accountsPath                     string
+	scratchDirectory                 string
+	rpcEndpoints                     []string
+	cluster                          string // "alpenglow", "mainnet-beta", "testnet", or "devnet"
+	legacyGenesisHash                string // explicit lineage for pre-binding AccountsDB/ledger artifacts
+	blockSource                      string // "turbine", "rpc", or "lightbringer"
+	lightbringerEndpoint             string
+	repairCatchupMaxGapSlots         int    // Resume gaps up to this fill via turbine repair instead of RPC (0 = off)
+	repairMaxRequestsPerSecond       int    // Repair request-rate ceiling override (0 = adaptive default)
+	blockRPCFallback                 bool   // Allow RPC block fetch when > repairCatchupMaxGapSlots behind (default false: shreds only)
+	blockMaxRPS                      int    // Rate limit for block fetching
+	blockMaxInflight                 int    // Max concurrent block fetch workers
+	blockTipPollIntervalMs           int    // Tip poll interval in milliseconds
+	blockTipSafetyMargin             int    // Don't fetch within N slots of tip
+	consensusModeFlag                string // raw --consensus-mode value (cobra binding)
+	consensusMode                    string // resolved: "verifying" (default) or "validator"
+	alpenglowObserverBindAddr        string
+	alpenglowMaxMessageBytes         int64
+	alpenglowBLSDST                  string
+	validatorIdentityKeypair         string
+	validatorVoteAccountKeypair      string
+	validatorAuthorizedVoterKeypair  string
+	validatorWithdrawerKeypair       string
+	validatorTPUQUICBind             string
+	validatorAdvertisedIP            string
+	validatorSigverifyWorkers        int
+	validatorCompletionReserveMs     int
+	validatorMaxBufferedTransactions int
 
 	// Mode thresholds
 	blockNearTipThreshold        int // Enter near-tip when gap <= this
@@ -547,6 +549,8 @@ func init() {
 	Run.Flags().StringVar(&validatorTPUQUICBind, "tpu-quic-bind-addr", "", "Validator TPU QUIC listen address (default 0.0.0.0:8004)")
 	Run.Flags().StringVar(&validatorAdvertisedIP, "validator-advertised-ip", "", "Public IP advertised for validator TPU QUIC")
 	Run.Flags().IntVar(&validatorSigverifyWorkers, "tpu-sigverify-workers", 0, "TPU signature verification workers (0 = GOMAXPROCS)")
+	Run.Flags().IntVar(&validatorCompletionReserveMs, "leader-completion-reserve-ms", 0, "Time reserved for leader finalization and broadcast (0 = 75ms default; tune from measured completion times)")
+	Run.Flags().IntVar(&validatorMaxBufferedTransactions, "tpu-max-buffered-transactions", 0, "Maximum queued TPU transactions (0 = 131072 default)")
 
 	// [tuning] section flags
 	Run.Flags().Uint64Var(&paramArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)")
@@ -869,6 +873,14 @@ func initConfigAndBindFlags(cmd *cobra.Command) error {
 	}
 	validatorAdvertisedIP = getString("validator-advertised-ip", "validator.advertised_ip")
 	validatorSigverifyWorkers = getInt("tpu-sigverify-workers", "validator.tpu_sigverify_workers")
+	validatorCompletionReserveMs = getInt("leader-completion-reserve-ms", "validator.block_completion_reserve_ms")
+	validatorMaxBufferedTransactions = getInt("tpu-max-buffered-transactions", "validator.tpu_max_buffered_transactions")
+	if validatorMaxBufferedTransactions < 0 {
+		return fmt.Errorf("TPU maximum buffered transactions must be nonnegative")
+	}
+	if validatorCompletionReserveMs < 0 || validatorCompletionReserveMs >= int(blockprod.AlpenglowSlotDuration/time.Millisecond) {
+		return fmt.Errorf("leader completion reserve must be 0 (default) or between 1 and 199 milliseconds")
+	}
 
 	// [block] section
 	blockSource = getString("block-source", "block.source")
@@ -2718,7 +2730,9 @@ postBootstrap:
 		defer broadcaster.Close()
 
 		controller := blockprod.NewController()
-		topicSink := scheduler.New(controller)
+		topicSink := scheduler.NewWithConfig(controller, scheduler.Config{
+			FeatureSource: replay.ChainTipFeatures, MaxBufferedTransactions: validatorMaxBufferedTransactions,
+		})
 		topicSink.Start(ctx)
 		defer topicSink.Stop()
 		tpuCfg := tpu.DefaultConfig()
@@ -2765,14 +2779,15 @@ postBootstrap:
 		leaderStop := make(chan struct{})
 		leaderDone := make(chan struct{})
 		leaderLoop := blockprod.NewLeaderLoop(blockprod.LeaderLoopConfig{
-			Controller:     controller,
-			Identity:       solana.PrivateKey(validatorIdentity),
-			AccountsDb:     accountsDb,
-			Broadcaster:    broadcaster,
-			ShredVersion:   uint16(turbineShredVersion),
-			EpochSchedule:  epochSchedule,
-			AlpenglowClock: true,
-			SlotDuration:   blockprod.AlpenglowSlotDuration,
+			Controller:        controller,
+			Identity:          solana.PrivateKey(validatorIdentity),
+			AccountsDb:        accountsDb,
+			Broadcaster:       broadcaster,
+			ShredVersion:      uint16(turbineShredVersion),
+			EpochSchedule:     epochSchedule,
+			AlpenglowClock:    true,
+			SlotDuration:      blockprod.AlpenglowSlotDuration,
+			CompletionReserve: time.Duration(validatorCompletionReserveMs) * time.Millisecond,
 			ParentContext: func(slot uint64) blockprod.ParentContext {
 				tip := replay.ChainTipParentContext()
 				// Blockprod owns the replay-readiness rule. In particular, the first

config.example.toml

@@ -328,6 +328,15 @@ name = "mithril"
     # Signature-verification workers (0 = GOMAXPROCS).
     tpu_sigverify_workers = 0
 
+    # Bounded cross-slot TPU queue. Zero keeps the 131,072-transaction default.
+    # Larger queues can prefill four busy leader slots, using additional memory.
+    tpu_max_buffered_transactions = 0
+
+    # Milliseconds reserved for local finalization and broadcast, not consensus
+    # finality. Zero keeps the conservative 75ms default. Tune from measured
+    # completion margins; this does not change the protocol slot deadline.
+    block_completion_reserve_ms = 0
+
 # ============================================================================
 # [consensus] - Alpenglow Consensus
 # ============================================================================

docs/leader-packing-evidence.md

@@ -0,0 +1,14 @@
+# Leader Packing: benchmark evidence
+
+The maintained subsystem documentation and reusable Go benchmarks describe the
+implementation and reproduction method. Historical raw results and session
+notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/06ef067798c99947e8cc527450ad28430a9a7333)
+(tag `review-evidence-20260916-leader-packing`). They are omitted from this proposed merge.
+
+[Historical result files](https://github.com/Overclock-Validator/mithril/tree/06ef067798c99947e8cc527450ad28430a9a7333/docs/results)
+
+Measurements retain their original baselines. Rebasing onto PR #278 does not
+turn an intermediate-version benchmark into a comparison with the new base.
+Component timings and short live observations do not establish sustained FAST
+inclusion gains. The final review description records validation of the rebased
+source separately from historical benchmark results.

docs/leader_block_packing.md

@@ -0,0 +1,98 @@
+# Leader block packing and synthetic load tests
+
+This change reduces work performed during a leader's available packing window.
+The scheduler owns and decodes packet bytes once, and prepares static message
+validation, instructions/account metadata, compute limits, message hash and cost
+while transactions are queued. Each bank checks the immutable feature snapshot
+before reuse. Account state, age, duplicates, strict fee-payer eligibility, rent,
+execution and all resource budgets remain bank-dependent checks.
+
+Leader execution reuses borrowed-account scratch and skips detailed replay
+stage timers. Missing-current-bank lookup errors defer base58 formatting until
+used, avoiding wasted work before parent lookup. Entry Merkle hashing retains
+only the root-building scratch, and max-heap removal avoids heap interface dispatch
+while preserving priority/FIFO order. Both priority heaps now track entry
+indexes so consumption, eviction and expiry remove every buffer reference.
+Repeated rebuffering reuses the caller's intact transaction without accumulating
+duplicate heap references. The existing slot-local skip scanning policy remains
+unchanged, including selection of newly arrived higher-priority transactions.
+
+## Capacity and protocol limits
+
+Live banks use slot-dependent budgets with slot-time feature gates taking effect
+in the epoch after activation. For the September 13 cluster's 200ms regime with
+RaiseBlockLimitsTo100m, the budget was 50M block-cost units, 20M writable-account
+cost units, 50MB allocated-data growth and 10MiB entry bytes. The entry packer
+reserves 48 bytes for the ending tick. A 100M per-slot budget would be incorrect
+in this regime. Active limits are logged when a leader bank opens.
+
+The readonly-pair fixture has one signature, one writable fee payer, two existing
+readonly accounts, no instructions, and 198 wire bytes. Its observed actual cost
+is 1,028 units: 720 signature, 300 write lock and eight loaded-account units. Its
+program execution cost is zero. The theoretical cost-only ceiling is 48,638,
+but upfront admission must fit the larger estimated loaded-data reservation:
+the offline bank test includes 48,622 before rejecting the next transaction.
+This workload is designed for signature/packing load, not application execution.
+
+## Reproduce locally
+
+All commands below are offline. They use deterministic test keys and in-memory
+accounts; no RPC, faucet, funding or transaction submission occurs.
+
+```sh
+# The 200,000-message, eight-payer, two-blockhash workload used to prefill four slots.
+go test ./pkg/tpu/txfixture -run '^TestReadonlyPair200KDistinctMessages$' -count=1
+
+# Fill a 50M-cost bank, reject the next tx, check fees, and round-trip all entries
+# through actual shred generation and decoding, preserving transaction order/hash.
+go test ./pkg/blockprod -run '^TestReadonlyPairBlockCapacityAndShredRoundTrip$' -count=1
+
+# Whole-bank construction: one serial caller; pre-signed unique transactions.
+GOMAXPROCS=8 go test ./pkg/blockprod -run '^$' \
+  -bench '^BenchmarkReadonlyPair(FullBlock|PreparedFullBlock)$' -benchtime=3x -count=3
+
+# More representative instruction workloads and smaller component microbenchmarks.
+GOMAXPROCS=8 go test ./pkg/blockprod -run '^$' \
+  -bench '^BenchmarkWorkingBank(Decoded|Prepared)?HotAccounts$' -benchtime=2s -count=3
+```
+
+The whole-bank benchmark admits 48,622 transactions and includes execution,
+account publication, entry batching/hash work and final entry flush. Signing and
+bank setup are excluded. The prepared variant additionally does static
+preparation before timing, modeling a queue ready before leadership. That work
+is moved, not eliminated. Actual AccountsDB, signature verification, network,
+consensus and the protocol deadline are outside this benchmark. The correctness
+test's shred round-trip is also outside the timed benchmark.
+
+`txfixture.ReadonlyPairWire` provides the same ordered-pair construction as the
+live experiment: 128×127 unique messages per payer/blockhash. Repeated ordinals
+need a different payer or blockhash. The 200k test verifies uniqueness across
+all messages and decodes/verifies representative signatures and phase boundaries.
+
+## Queue supply and completion reserve
+
+```toml
+[validator]
+tpu_max_buffered_transactions = 0 # default 131072
+block_completion_reserve_ms = 0  # default 75ms
+```
+
+The corresponding flags are `--tpu-max-buffered-transactions` and
+`--leader-completion-reserve-ms`. The measured full-prefill trial used 262,144
+queue entries and a 60ms reserve. Those are opt-in tuning values; defaults stay
+unchanged. A larger queue uses additional memory for owned wire, decoded and
+prepared objects. A shorter reserve needs measured local finalization/broadcast
+margin and does not change the protocol deadline. Shifting completion also
+shifts later bank start times, so it does not add the same packing time to all
+four blocks.
+
+The live results showed why total supply and timing matter: 120k transactions
+cannot fill four approximately 48.6k blocks. An 80k refill competed with ongoing
+work. Preloading 200k into a larger queue improved the observed four-block total,
+while the first block still had less usable time and later banks awaited local
+replay/adoption. These observations do not isolate a single CPU bottleneck.
+
+[Measured results, exact baselines and evidence](https://github.com/Overclock-Validator/mithril/blob/06ef067798c99947e8cc527450ad28430a9a7333/docs/results/leader-block-packing/2026-09-13/README.md)
+include both the successful near-limit block and the still-underfilled four-slot
+window. The archived live helper is historical experiment source with explicit
+cluster/identity/path constants; the offline fixture is the portable reproduction.

pkg/accounts/mem_accounts.go

@@ -1,7 +1,6 @@
 package accounts
 
 import (
-	"fmt"
 	"sync"
 
 	"github.com/Overclock-Validator/mithril/pkg/base58"
@@ -13,6 +12,16 @@ type MemAccounts struct {
 	mu  *sync.RWMutex
 }
 
+// A miss is an ordinary step when falling back to parent accounts. Defer the
+// diagnostic encoding until it is needed, and copy the key so callers can reuse it.
+type missingMemAccountError struct {
+	key [32]byte
+}
+
+func (e *missingMemAccountError) Error() string {
+	return "no such account " + base58.Encode(e.key[:]) + " found"
+}
+
 func NewMemAccounts() MemAccounts {
 	return MemAccounts{
 		Map: make(map[[32]byte]*Account),
@@ -32,15 +41,15 @@ func (m MemAccounts) GetAccount(pubkey *[32]byte) (*Account, error) {
 	defer m.mu.RUnlock()
 	acct, ok := m.Map[*pubkey]
 	if !ok {
-		return nil, fmt.Errorf("no such account %s found", base58.Encode(pubkey[:]))
+		return nil, &missingMemAccountError{key: *pubkey}
 	}
 	return acct, nil
 }
 
 func (m MemAccounts) GetAccountWithoutLock(pubkey solana.PublicKey) (*Account, error) {
 	acct, ok := m.Map[pubkey]
 	if !ok {
-		return nil, fmt.Errorf("no such account %s found", base58.Encode(pubkey[:]))
+		return nil, &missingMemAccountError{key: pubkey}
 	}
 	return acct, nil
 }

pkg/accounts/mem_accounts_test.go

@@ -3,8 +3,23 @@ package accounts
 import (
 	"testing"
 	"time"
+
+	"github.com/gagliardetto/solana-go"
 )
 
+func TestMemAccountMissingErrorRetainsLookupKey(t *testing.T) {
+	mem := NewMemAccounts()
+	key := [32]byte{}
+	_, lockedErr := mem.GetAccount(&key)
+	_, unlockedErr := mem.GetAccountWithoutLock(solana.PublicKey(key))
+	key[0] = 99 // Lookup callers may reuse their key storage before reporting an error.
+	for _, err := range []error{lockedErr, unlockedErr} {
+		if err == nil || err.Error() != "no such account 11111111111111111111111111111111 found" {
+			t.Fatalf("missing error lost its original key: %v", err)
+		}
+	}
+}
+
 func TestMemAccountsReadsAreConcurrent(t *testing.T) {
 	mem := NewMemAccounts()
 	var key [32]byte