diff --git a/cmd/mithril/configcmd/configcmd.go b/cmd/mithril/configcmd/configcmd.go index 903ff68ca..3236b9fcc 100644 --- a/cmd/mithril/configcmd/configcmd.go +++ b/cmd/mithril/configcmd/configcmd.go @@ -261,7 +261,12 @@ max_rps = 8 # Verifier's own RPC budget (never shares the block-fe # ── Replay tuning ──────────────────────────────────────────────────────── [tuning] txpar = 24 # Validator auto-defaults to 2x CPU cores only when unset; explicit 0 = sequential -sigverify_backend = "auto" # auto|r51|generic|stdlib; stdlib uses Go's crypto/ed25519 impl after strict checks. + +[sigverify] +backend = "auto" # auto|r51|generic|stdlib +workers = 0 # 0 = min(2, GOMAXPROCS); explicit value overrides the shared transaction pool +batch_target = 8 # 4 or 8 signature lanes; available work runs immediately +disable_shred_overlap = false # Diagnostic fallback: verify after complete block assembly # ── Mithril's RPC server ───────────────────────────────────────────────── [rpc] diff --git a/cmd/mithril/configcmd/configcmd_test.go b/cmd/mithril/configcmd/configcmd_test.go new file mode 100644 index 000000000..2d62bb8b5 --- /dev/null +++ b/cmd/mithril/configcmd/configcmd_test.go @@ -0,0 +1,24 @@ +package configcmd + +import ( + "strings" + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" +) + +func TestStarterConfigSignatureVerification(t *testing.T) { + for _, validator := range []bool{false, true} { + v := viper.New() + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(strings.NewReader(generateStarterConfig(validator)))) + require.Equal(t, "auto", v.GetString("sigverify.backend")) + require.True(t, v.IsSet("sigverify.workers")) + require.Zero(t, v.GetInt("sigverify.workers")) + require.Equal(t, 8, v.GetInt("sigverify.batch_target")) + require.True(t, v.IsSet("sigverify.disable_shred_overlap")) + require.False(t, v.GetBool("sigverify.disable_shred_overlap")) + require.False(t, v.IsSet("tuning.sigverify_backend")) + } +} diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index c452d494e..a7c95f9e2 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -74,35 +74,37 @@ var ( }, } - bootstrapMode string // "auto", "snapshot", "new-snapshot", "new-incremental", or "accountsdb" - snapshotArchivePath string - incrementalSnapshotFilename string - accountsPath string - scratchDirectory string - rpcEndpoints []string - cluster string // "alpenglow", "mainnet-beta", "testnet", or "devnet" - legacyGenesisHash string // explicit lineage for pre-binding AccountsDB/ledger artifacts - blockSource string // "turbine", "rpc", or "lightbringer" - lightbringerEndpoint string - repairCatchupMaxGapSlots int // Resume gaps up to this fill via turbine repair instead of RPC (0 = off) - repairMaxRequestsPerSecond int // Repair request-rate ceiling override (0 = adaptive default) - blockRPCFallback bool // Allow RPC block fetch when > repairCatchupMaxGapSlots behind (default false: shreds only) - blockMaxRPS int // Rate limit for block fetching - blockMaxInflight int // Max concurrent block fetch workers - blockTipPollIntervalMs int // Tip poll interval in milliseconds - blockTipSafetyMargin int // Don't fetch within N slots of tip - consensusModeFlag string // raw --consensus-mode value (cobra binding) - consensusMode string // resolved: "verifying" (default) or "validator" - alpenglowObserverBindAddr string - alpenglowMaxMessageBytes int64 - alpenglowBLSDST string - validatorIdentityKeypair string - validatorVoteAccountKeypair string - validatorAuthorizedVoterKeypair string - validatorWithdrawerKeypair string - validatorTPUQUICBind string - validatorAdvertisedIP string - validatorSigverifyWorkers int + bootstrapMode string // "auto", "snapshot", "new-snapshot", "new-incremental", or "accountsdb" + snapshotArchivePath string + incrementalSnapshotFilename string + accountsPath string + scratchDirectory string + rpcEndpoints []string + cluster string // "alpenglow", "mainnet-beta", "testnet", or "devnet" + legacyGenesisHash string // explicit lineage for pre-binding AccountsDB/ledger artifacts + blockSource string // "turbine", "rpc", or "lightbringer" + lightbringerEndpoint string + repairCatchupMaxGapSlots int // Resume gaps up to this fill via turbine repair instead of RPC (0 = off) + repairMaxRequestsPerSecond int // Repair request-rate ceiling override (0 = adaptive default) + blockRPCFallback bool // Allow RPC block fetch when > repairCatchupMaxGapSlots behind (default false: shreds only) + blockMaxRPS int // Rate limit for block fetching + blockMaxInflight int // Max concurrent block fetch workers + blockTipPollIntervalMs int // Tip poll interval in milliseconds + blockTipSafetyMargin int // Don't fetch within N slots of tip + consensusModeFlag string // raw --consensus-mode value (cobra binding) + consensusMode string // resolved: "verifying" (default) or "validator" + alpenglowObserverBindAddr string + alpenglowMaxMessageBytes int64 + alpenglowBLSDST string + validatorIdentityKeypair string + validatorVoteAccountKeypair string + validatorAuthorizedVoterKeypair string + validatorWithdrawerKeypair string + validatorTPUQUICBind string + validatorAdvertisedIP string + validatorSigverifyWorkers int + validatorCompletionReserveMs int + validatorMaxBufferedTransactions int // Mode thresholds blockNearTipThreshold int // Enter near-tip when gap <= this @@ -547,6 +549,8 @@ func init() { Run.Flags().StringVar(&validatorTPUQUICBind, "tpu-quic-bind-addr", "", "Validator TPU QUIC listen address (default 0.0.0.0:8004)") Run.Flags().StringVar(&validatorAdvertisedIP, "validator-advertised-ip", "", "Public IP advertised for validator TPU QUIC") Run.Flags().IntVar(&validatorSigverifyWorkers, "tpu-sigverify-workers", 0, "TPU signature verification workers (0 = GOMAXPROCS)") + Run.Flags().IntVar(&validatorCompletionReserveMs, "leader-completion-reserve-ms", 0, "Time reserved for leader finalization and broadcast (0 = 75ms default; tune from measured completion times)") + Run.Flags().IntVar(&validatorMaxBufferedTransactions, "tpu-max-buffered-transactions", 0, "Maximum queued TPU transactions (0 = 131072 default)") // [tuning] section flags Run.Flags().Uint64Var(¶mArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)") @@ -560,6 +564,12 @@ func init() { Run.Flags().StringVar(&snapshot.SnapshotIndexTempDir, "snapshot-index-temp-dir", "", "Optional directory for snapshot index shard logs/SST staging") Run.Flags().StringVar(&sigverify.Cfg.Backend, "sigverify-backend", sigverify.Defaults().Backend, "ed25519 verification backend: auto|r51|generic|stdlib") + Run.Flags().IntVar(&sigverify.Cfg.Workers, "sigverify-workers", 0, + "Turbine transaction signature verification workers (0 = min(2, GOMAXPROCS))") + Run.Flags().IntVar(&sigverify.Cfg.BatchTarget, "sigverify-batch-target", sigverify.Defaults().BatchTarget, + "Turbine transaction signature batch target: 4 or 8 (available short batches run immediately)") + Run.Flags().BoolVar(&sigverify.Cfg.DisableShredOverlap, "sigverify-disable-shred-overlap", false, + "Defer Turbine transaction decoding and signature verification until all block shreds arrive") Run.Flags().BoolVar(&sbpf.UsePool, "use-pool", true, "Disable to allocate fresh slices") Run.Flags().IntVar(&accountsdb.StoreAccountsWorkers, "store-accounts-workers", 128, "Number of workers to write account updates") Run.Flags().IntVar(&accountsdb.ProgramCacheMaxMB, "program-cache-max-mb", accountsdb.DefaultProgramCacheMaxMB, "Maximum approximate SBPF program cache size in MiB") @@ -863,6 +873,14 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { } validatorAdvertisedIP = getString("validator-advertised-ip", "validator.advertised_ip") validatorSigverifyWorkers = getInt("tpu-sigverify-workers", "validator.tpu_sigverify_workers") + validatorCompletionReserveMs = getInt("leader-completion-reserve-ms", "validator.block_completion_reserve_ms") + validatorMaxBufferedTransactions = getInt("tpu-max-buffered-transactions", "validator.tpu_max_buffered_transactions") + if validatorMaxBufferedTransactions < 0 { + return fmt.Errorf("TPU maximum buffered transactions must be nonnegative") + } + if validatorCompletionReserveMs < 0 || validatorCompletionReserveMs >= int(blockprod.AlpenglowSlotDuration/time.Millisecond) { + return fmt.Errorf("leader completion reserve must be 0 (default) or between 1 and 199 milliseconds") + } // [block] section blockSource = getString("block-source", "block.source") @@ -1127,10 +1145,17 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { // later: narya pins its backend on first use, and selecting it explicitly // doubles as a startup health check, so a machine that cannot run the // requested backend fails now instead of at the first block. - sigverify.Cfg.Backend = getString("sigverify-backend", "tuning.sigverify_backend") + backendKey := "sigverify.backend" + if !config.IsSet(backendKey) { + backendKey = "tuning.sigverify_backend" // older configuration files + } + sigverify.Cfg.Backend = getString("sigverify-backend", backendKey) + sigverify.Cfg.Workers = getInt("sigverify-workers", "sigverify.workers") + sigverify.Cfg.BatchTarget = getInt("sigverify-batch-target", "sigverify.batch_target") + sigverify.Cfg.DisableShredOverlap = getBool("sigverify-disable-shred-overlap", "sigverify.disable_shred_overlap") resolved, err := sigverify.Configure(sigverify.Cfg) if err != nil { - return fmt.Errorf("tuning.sigverify_backend: %w", err) + return fmt.Errorf("signature verification configuration: %w", err) } resolvedSigverifyBackend = resolved sbpf.UsePool = getBool("use-pool", "tuning.use_pool") @@ -2705,7 +2730,9 @@ postBootstrap: defer broadcaster.Close() controller := blockprod.NewController() - topicSink := scheduler.New(controller) + topicSink := scheduler.NewWithConfig(controller, scheduler.Config{ + FeatureSource: replay.ChainTipFeatures, MaxBufferedTransactions: validatorMaxBufferedTransactions, + }) topicSink.Start(ctx) defer topicSink.Stop() tpuCfg := tpu.DefaultConfig() @@ -2752,14 +2779,15 @@ postBootstrap: leaderStop := make(chan struct{}) leaderDone := make(chan struct{}) leaderLoop := blockprod.NewLeaderLoop(blockprod.LeaderLoopConfig{ - Controller: controller, - Identity: solana.PrivateKey(validatorIdentity), - AccountsDb: accountsDb, - Broadcaster: broadcaster, - ShredVersion: uint16(turbineShredVersion), - EpochSchedule: epochSchedule, - AlpenglowClock: true, - SlotDuration: blockprod.AlpenglowSlotDuration, + Controller: controller, + Identity: solana.PrivateKey(validatorIdentity), + AccountsDb: accountsDb, + Broadcaster: broadcaster, + ShredVersion: uint16(turbineShredVersion), + EpochSchedule: epochSchedule, + AlpenglowClock: true, + SlotDuration: blockprod.AlpenglowSlotDuration, + CompletionReserve: time.Duration(validatorCompletionReserveMs) * time.Millisecond, ParentContext: func(slot uint64) blockprod.ParentContext { tip := replay.ChainTipParentContext() // Blockprod owns the replay-readiness rule. In particular, the first @@ -3214,6 +3242,8 @@ func printStartupInfo(commandName string) { } fmt.Printf(" Sigverify: %s%s%s %s(%s)%s\n", green, resolvedSigverifyBackend, reset, dim, sigverifyDesc, reset) + fmt.Printf(" workers=%d batch_target=%d shred_overlap=%t\n", + sigverify.TransactionWorkers(), sigverify.TransactionBatchTarget(), !sigverify.Cfg.DisableShredOverlap) } // Load state file for detailed info (only show for modes that use existing AccountsDB) diff --git a/cmd/mithril/node/sigverify_reporter.go b/cmd/mithril/node/sigverify_reporter.go index d374d93b9..597a3c116 100644 --- a/cmd/mithril/node/sigverify_reporter.go +++ b/cmd/mithril/node/sigverify_reporter.go @@ -42,7 +42,8 @@ func startSigverifyReporter(ctx context.Context) { // against, and so the resolved backend is recorded even on a node that // exits before the first tick. previous := sigverify.Stats() - mlog.NamedFilef("sigverify", "startup: %s", previous) + mlog.NamedFilef("sigverify", "startup: %s workers=%d batch_target=%d shred_overlap=%t", previous, + sigverify.TransactionWorkers(), sigverify.TransactionBatchTarget(), !sigverify.Cfg.DisableShredOverlap) for { select { diff --git a/cmd/repair-sim/main.go b/cmd/repair-sim/main.go new file mode 100644 index 000000000..5eebf05e9 --- /dev/null +++ b/cmd/repair-sim/main.go @@ -0,0 +1,137 @@ +// repair-sim runs deterministic, single-node Turbine repair scenarios. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/Overclock-Validator/mithril/pkg/turbine/repairsim" +) + +type environment struct { + GoVersion string `json:"go_version"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + CPU string `json:"cpu"` +} + +type report struct { + Environment environment `json:"environment"` + Ledger repairsim.LedgerConfig `json:"ledger"` + Network repairsim.Config `json:"network"` + LedgerGenerationWall time.Duration `json:"ledger_generation_wall_ns"` + Result repairsim.Result `json:"result"` +} + +func main() { + var ( + scenarioFlag = flag.String("scenario", string(repairsim.ScenarioNearTip), "near-tip or deep-catchup") + slots = flag.Int("slots", 200, "number of deterministic slots") + fecSets = flag.Int("fec-sets", 4, "FEC sets generated per slot") + entries = flag.Int("entries", 0, "entries per slot (0 derives an exact FEC count)") + seed = flag.Int64("seed", 1, "deterministic content and network seed") + availability = flag.String("availability", "", "complete, near-loss, sparse, or mixed") + repair = flag.Bool("repair", true, "enable repair requests") + latency = flag.Duration("repair-latency", 20*time.Millisecond, "synthetic one-way response latency") + jitter = flag.Duration("repair-jitter", 2*time.Millisecond, "deterministic +/- response jitter") + loss = flag.Float64("packet-loss", 0, "repair response loss probability [0,1]") + duplicates = flag.Float64("duplicates", 0.02, "duplicate response probability [0,1]") + bandwidth = flag.Int64("repair-bandwidth", 100*1024*1024, "synthetic repair bytes/sec (0 is unlimited)") + concurrent = flag.Int("max-concurrent", 256, "maximum outstanding repair shreds") + corrupt = flag.Int("corrupt-responses", 0, "corrupt the first N repair responses") + naturalLate = flag.Bool("natural-late", true, "schedule selected late live shreds during repair") + spoolDir = flag.String("spool-dir", "", "persistent shred-spool directory (empty uses a temporary directory)") + cpuLabel = flag.String("cpu-label", "", "explicit CPU label when platform discovery is unavailable") + output = flag.String("output", "", "write JSON to this file instead of stdout") + includeTrace = flag.Bool("trace", true, "include the logical event trace in JSON") + ) + flag.Parse() + + scenario := repairsim.Scenario(*scenarioFlag) + network := repairsim.DefaultConfig(scenario) + network.Availability = repairsim.Availability(*availability) + network.RepairEnabled = *repair + network.RepairLatency = *latency + network.RepairJitter = *jitter + network.PacketLoss = *loss + network.DuplicateProbability = *duplicates + network.BandwidthBytesPerSec = *bandwidth + network.MaxConcurrent = *concurrent + network.CorruptResponses = *corrupt + network.NaturalLateShreds = *naturalLate + network.CollectTrace = *includeTrace + network.Seed = *seed + network.SpoolDir = *spoolDir + if network.Availability == "" { + network.Availability = repairsim.DefaultConfig(scenario).Availability + } + + ledgerCfg := repairsim.LedgerConfig{ + StartSlot: 10_000, + Slots: *slots, + FECsPerSlot: *fecSets, + EntriesPerSlot: *entries, + Seed: *seed, + ShredVersion: 1, + ReferenceTick: 63, + } + started := time.Now() + ledger, err := repairsim.GenerateLedger(ledgerCfg) + if err != nil { + fatalf("generate ledger: %v", err) + } + generationWall := time.Since(started) + result, err := repairsim.Run(ledger, network) + if err != nil { + fatalf("run simulation: %v", err) + } + cpu := *cpuLabel + if cpu == "" { + cpu = cpuModel() + } + report := report{ + Environment: environment{GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, CPU: cpu}, + Ledger: ledger.Config, Network: network, LedgerGenerationWall: generationWall, Result: result, + } + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + fatalf("marshal report: %v", err) + } + encoded = append(encoded, '\n') + if *output == "" { + _, _ = os.Stdout.Write(encoded) + return + } + if err := os.WriteFile(*output, encoded, 0o644); err != nil { + fatalf("write %s: %v", *output, err) + } +} + +func cpuModel() string { + if runtime.GOOS == "linux" { + if data, err := os.ReadFile("/proc/cpuinfo"); err == nil { + for _, line := range strings.Split(string(data), "\n") { + if key, value, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "model name" { + return strings.TrimSpace(value) + } + } + } + } + if runtime.GOOS == "darwin" { + if out, err := exec.Command("sysctl", "-n", "machdep.cpu.brand_string").Output(); err == nil { + return strings.TrimSpace(string(out)) + } + } + return "unknown" +} + +func fatalf(format string, args ...any) { + _, _ = fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/config.example.toml b/config.example.toml index 7e3833087..2cf1affb9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -328,6 +328,15 @@ name = "mithril" # Signature-verification workers (0 = GOMAXPROCS). tpu_sigverify_workers = 0 + # Bounded cross-slot TPU queue. Zero keeps the 131,072-transaction default. + # Larger queues can prefill four busy leader slots, using additional memory. + tpu_max_buffered_transactions = 0 + + # Milliseconds reserved for local finalization and broadcast, not consensus + # finality. Zero keeps the conservative 75ms default. Tune from measured + # completion margins; this does not change the protocol slot deadline. + block_completion_reserve_ms = 0 + # ============================================================================ # [consensus] - Alpenglow Consensus # ============================================================================ @@ -524,14 +533,6 @@ name = "mithril" # Number of borrowed accounts to preallocate in arena (0 to disable) borrowed_account_arena_size = 1024 - # ed25519 signature verification backend. - # auto - use the AVX-512 accelerated backend when the CPU has - # AVX512-IFMA (Zen 4/5, Ice Lake and newer), else portable - # r51 - force the accelerated backend; startup fails without AVX512-IFMA - # generic - force the portable pure-Go backend - # stdlib - use Go’s crypto/ed25519 implementation after the mandatory strict rejection checks. - sigverify_backend = "auto" - # Enable/disable pool allocator for slices use_pool = true @@ -574,6 +575,33 @@ name = "mithril" # Filename to write CPU profile (for offline analysis with go tool pprof) # cpu_profile_path = "/mnt/mithril-data/profiling/cpu.pprof" +# ============================================================================ +# [sigverify] - Transaction Signature Verification +# ============================================================================ + +[sigverify] + # ed25519 backend: auto selects AVX-512 IFMA when available, else portable. + # r51 requires AVX-512 IFMA; generic and stdlib force portable backends. + # Strict signature checks are always enabled. The older + # tuning.sigverify_backend key remains supported when this key is absent. + backend = "auto" + + # Shared Turbine transaction verification workers; 0 = min(2, GOMAXPROCS). + # Leaves execution and shred/consensus processing room to run concurrently. + # This does not change validator.tpu_sigverify_workers or replay's fallback pool. + workers = 0 + + # Signature lanes per group: 4 or 8 (0 also means 8). Transactions stay + # indivisible, so a multisignature transaction may exceed this target. + # Ready short groups run immediately; no timer waits for more shreds. + batch_target = 8 + + # Decode complete entry batches and verify while later shreds arrive. + # Set true to compare against completion-only verification. + # Early work reserves at most 8 slots and 64 MiB of encoded component bytes; + # decoded transactions and Go bookkeeping use additional heap memory. + disable_shred_overlap = false + # ============================================================================ # [debug] - Debug Logging # ============================================================================ diff --git a/docs/erasure_recovery_experiments.md b/docs/erasure_recovery_experiments.md new file mode 100644 index 000000000..5c47fe2fe --- /dev/null +++ b/docs/erasure_recovery_experiments.md @@ -0,0 +1,130 @@ +# Fixed-shape FEC recovery + +Production uses direct recovery only for exactly one missing data shard in a +32-data/32-coding FEC set with an available coding shard. All other availability +patterns keep the general decoder. Recovery still passes ordinary packet/root +validation before assembler admission. + +The deterministic production-path harness is described in [repair_sim.md](repair_sim.md). +Reduced-subset and all-coding plans remain reference benchmarks, not runtime +policies. Historical investigation notes are in the [evidence archive](fec-producer-evidence.md). + +## Matrix contract + +The experiment uses the systematic generator over `GF(256)/0x11d`: + +```text +V[x,j] = x^j +A = V[0:32,0:32] +G = V * A^-1 +G = [I_32; C] +``` + +For the fixed 32+32 shape, exhaustive scalar tests confirm all 1,024 entries: + +```text +C[r,c] = 0xa5 / (0x20 xor r xor c) +``` + +They also confirm `C*C=I`. Recovery tests remain differential against +`github.com/klauspost/reedsolomon`; the closed form is not the sole oracle. + +## Candidates + +### Near tip: direct one-data recovery + +For missing data position `m` and available coding position `r`: + +```text +D_m = C[r,m]^-1 * P_r + + sum(i != m, C[r,m]^-1 * C[r,i] * D_i) +``` + +This prepares one 32-source coefficient row and writes one destination. It does +not construct or invert a general 32x32 matrix. + +Production uses a process-wide table containing every missing-data and +coding-row combination. This removes per-call plan construction while keeping +the same equation. The table is exhaustively differential-tested against the +general decoder across all 32 x 32 combinations. + +### Catch up: reduced missing-data system + +For missing data columns `M` and selected coding rows `R`, substitute every +known data shard and solve: + +```text +B[i,j] = C[R[i],M[j]] +B * D_M = adjusted_coding_rows +``` + +The experiment uses the Cauchy closed form to construct `B^-1` in `O(m^2)`, +expands direct rows over 32 selected sources, and proves each row satisfies the +requested systematic generator row before byte processing. An independently +implemented Gauss-Jordan inverse is the setup fallback. + +The byte kernel is intentionally portable and uses `reedsolomon.LowLevel`. +This isolates algorithm and plan costs; it is not evidence that a portable +kernel will beat the dependency's generated AVX2/GFNI kernels on amd64. + +### Catch up edge: all coding rows + +When all data rows are missing and every coding row is present, `C*C=I` means +the existing optimized encoder can apply `C` to the coding rows and recover the +data directly. This is kept as a separate synthetic arm. It is simpler than a +general decoder, but a cached reduced-system plan may still have a faster byte +kernel; hardware decides between them. + +## Synthetic coverage + +The tests cover: + +- every missing-data position with every coding-row choice for the direct path; +- every pair of missing data positions; +- deterministic mixed patterns at 2, 4, 8, 16, 24, and 32 missing data shreds; +- exactly-threshold and one-below-threshold availability; +- changed availability between setup and execution; +- destination failure atomicity; +- coefficient mutation detection; +- Cauchy inverses against independent Gauss-Jordan inversion; +- recovered bytes against the existing general decoder. + +Run: + +```bash +go test ./pkg/turbine/internal/rsrecover +go test -run '^$' \ + -bench '^(BenchmarkRecoverOneData|BenchmarkRecoverDataSubset)$' \ + -benchmem -benchtime=2s -count=6 \ + ./pkg/turbine/internal/rsrecover +``` + +Benchmark result interpretation must keep these cases separate: + +- `prepare`: cold or changing erasure pattern; +- `execute`: prepared/repeated pattern; +- `prepare-and-execute`: first useful output for a new pattern; +- general cache off: existing decoder with changing pattern cost; +- general cache on: existing decoder after the inversion is cached. + +No production dispatch threshold should be chosen from an Apple benchmark. +Final crossover decisions require the pinned amd64 target and synthetic arrival +traces for progressing, stalled, and bursty slots. + +## Zen 5 production gate + +The direct one-data path was measured on a Ryzen 7 9700X with Go 1.26.4, +`GOMAXPROCS=1`, and one pinned physical core. Medians below are from seven +sequential one-second samples unless otherwise noted. + +| Benchmark | General path | Direct one-data path | Change | +| --- | ---: | ---: | ---: | +| one-missing `SlotAssembler` boundary | 10.73 us/FEC | 2.82 us/FEC | -73.7% (3.8x) | +| near-tip repair simulation | 3.0295 ms/op | 2.8659 ms/op | -5.40% | +| deep-mixed repair simulation | 4.0343 ms/op | 4.0240 ms/op | -0.26% | +| deep-sparse repair simulation | 10.8489 ms/op | 10.8327 ms/op | -0.15% | + +The production dispatch is intentionally narrow. The deep scenarios do not +enter it and remain effectively neutral, while the near-tip workload benefits +from repeated exactly-one-missing recoveries. The one-missing boundary also +dropped from 144 to 5 allocations per operation. diff --git a/docs/fec-producer-evidence.md b/docs/fec-producer-evidence.md new file mode 100644 index 000000000..68c57c7d2 --- /dev/null +++ b/docs/fec-producer-evidence.md @@ -0,0 +1,18 @@ +# FEC producer and recovery evidence + +The original #259 source, standalone producer benchmarks, raw samples and +recovery derivation are preserved at +[the historical snapshot](https://github.com/Overclock-Validator/mithril/tree/a3b16ebaaf803807ad04a7975f3eccf1c15649ea) +(tag `review-evidence-20260916-fec-producer`). + +The September 6 comparison used development head `7e4e8af1`, not today's #278 +base. Fifty thousand 1,232-byte legacy transactions across three slots took +734.052 → 282.296 ms on one pinned Zen 5 CPU. With both versions using the same +30,816-byte batch target, the result was 734.052 → 288.378 ms. This measures +serial producer work, excluding transaction execution, admission verification, +worker queue overlap, routing and network delivery. It is not a whole-validator +speedup or a benchmark of the rebased combined Turbine review. + +The rebase preserves newer slot-byte reservations and the asynchronous shred +worker. Fixtures explicitly use the legacy 1,232-byte limit rather than the +newer 4,096-byte transport maximum. Reusable benchmark code remains in source. diff --git a/docs/fec-recovery-authentication.md b/docs/fec-recovery-authentication.md new file mode 100644 index 000000000..a1b7b752b --- /dev/null +++ b/docs/fec-recovery-authentication.md @@ -0,0 +1,63 @@ +# Authenticating recovered FEC data + +Received shreds must pass leader-signature verification before entering the slot +assembler. Reed–Solomon reconstruction alone does not authenticate missing data: +a leader can sign a Merkle tree containing inconsistent data and coding shards. +Structural validation of recovered headers does not reject that case. + +Both the specialized one-missing-data decoder and the general decoder now require +`authenticateRecoveredFEC` to succeed before returning any recovered data. It +reconstructs missing coding shards too, builds the complete data/coding Merkle +tree, and compares its root with a received coding shred's signed root. Checking +only recovered-data proofs would not detect an inconsistent commitment to a +missing coding shard. Existing received packets are read-only throughout recovery. + +On success, recovered data receives complete Merkle proofs and the coding +template's chained root and, where applicable, retransmitter signature. The latter +is a hop signature copied from a received packet, not a reconstruction of a lost +relay's signature. On failure, no recovered data is published. The all-data-present +path does not reconstruct or authenticate another tree; it relies on ingress +verification of the received data. + +This follows [Agave's recovery algorithm](https://github.com/anza-xyz/alpenglow/blob/9f284c913f3c78b36179ae2461fa91286a616fb9/ledger/src/shred/merkle.rs#L670): +reconstruct all missing shards, validate recovered headers, compare the full root, +and populate proofs. Signature verification is not repeated after the root match. + +## Validation + +`TestRecoveredFECAuthentication` exercises one, three and all 32 missing data +shreds, chained and resigned packets, altered recovered bytes, and leader-signed +inconsistent parity both received and absent. Invalid sets return no recovered +shreds. Valid recovered packets verify with the leader's public key. + +`TestRecoveredFECAgaveSignedCapture` uses four FEC sets from the existing captured +Agave slot 1,752,420. It regenerates parity without signing a new root, checks the +original committed roots, and compares recovered authenticated bytes and proofs +with the capture. Transport repair nonces and relay-specific signatures are +handled separately. This is a captured-wire compatibility test, not a run of a +Rust recovery oracle. The older localnet recovery test also covers an unchained +1+17 layout, with its 2022 proofs replaced by current signed proofs. + +## Cost and reproduction + +Apple M4 Pro, `GOMAXPROCS=2`, five 200 ms samples, warmed encoder cache. The baseline +is `039ebd67`; times are medians for one FEC recovery, excluding ingress signature +verification and block execution. + +| Received / recovered shape | Before | Authenticated recovery | +|---|---:|---:| +| 31 data + 32 coding; recover one data | 2.47 µs | 30.17 µs | +| 31 data + 1 coding; recover one data and missing parity | — | 104.52 µs | +| 29 data + 3 coding; recover three data and missing parity | — | 117.93 µs | +| 32 coding; recover all data | — | 118.01 µs | + +The first case increases from 4,264 bytes / 5 allocations to 8,360 bytes / 6 +allocations per operation. Threshold-arrival cases also pay to reconstruct missing +parity and assemble its leaf bytes. These measurements establish the cost of the +correctness check, not a live FAST-score or whole-block performance result. + +``` +GOMAXPROCS=2 go test ./pkg/turbine -run '^$' \ + -bench 'Benchmark(RecoverFECOneMissingBoundary|AuthenticatedFECRecovery)$' \ + -benchmem -benchtime=200ms -count=5 +``` diff --git a/docs/leader-packing-evidence.md b/docs/leader-packing-evidence.md new file mode 100644 index 000000000..8a439d1eb --- /dev/null +++ b/docs/leader-packing-evidence.md @@ -0,0 +1,14 @@ +# Leader Packing: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/06ef067798c99947e8cc527450ad28430a9a7333) +(tag `review-evidence-20260916-leader-packing`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/06ef067798c99947e8cc527450ad28430a9a7333/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/docs/leader_block_packing.md b/docs/leader_block_packing.md new file mode 100644 index 000000000..fd76fe466 --- /dev/null +++ b/docs/leader_block_packing.md @@ -0,0 +1,98 @@ +# Leader block packing and synthetic load tests + +This change reduces work performed during a leader's available packing window. +The scheduler owns and decodes packet bytes once, and prepares static message +validation, instructions/account metadata, compute limits, message hash and cost +while transactions are queued. Each bank checks the immutable feature snapshot +before reuse. Account state, age, duplicates, strict fee-payer eligibility, rent, +execution and all resource budgets remain bank-dependent checks. + +Leader execution reuses borrowed-account scratch and skips detailed replay +stage timers. Missing-current-bank lookup errors defer base58 formatting until +used, avoiding wasted work before parent lookup. Entry Merkle hashing retains +only the root-building scratch, and max-heap removal avoids heap interface dispatch +while preserving priority/FIFO order. Both priority heaps now track entry +indexes so consumption, eviction and expiry remove every buffer reference. +Repeated rebuffering reuses the caller's intact transaction without accumulating +duplicate heap references. The existing slot-local skip scanning policy remains +unchanged, including selection of newly arrived higher-priority transactions. + +## Capacity and protocol limits + +Live banks use slot-dependent budgets with slot-time feature gates taking effect +in the epoch after activation. For the September 13 cluster's 200ms regime with +RaiseBlockLimitsTo100m, the budget was 50M block-cost units, 20M writable-account +cost units, 50MB allocated-data growth and 10MiB entry bytes. The entry packer +reserves 48 bytes for the ending tick. A 100M per-slot budget would be incorrect +in this regime. Active limits are logged when a leader bank opens. + +The readonly-pair fixture has one signature, one writable fee payer, two existing +readonly accounts, no instructions, and 198 wire bytes. Its observed actual cost +is 1,028 units: 720 signature, 300 write lock and eight loaded-account units. Its +program execution cost is zero. The theoretical cost-only ceiling is 48,638, +but upfront admission must fit the larger estimated loaded-data reservation: +the offline bank test includes 48,622 before rejecting the next transaction. +This workload is designed for signature/packing load, not application execution. + +## Reproduce locally + +All commands below are offline. They use deterministic test keys and in-memory +accounts; no RPC, faucet, funding or transaction submission occurs. + +```sh +# The 200,000-message, eight-payer, two-blockhash workload used to prefill four slots. +go test ./pkg/tpu/txfixture -run '^TestReadonlyPair200KDistinctMessages$' -count=1 + +# Fill a 50M-cost bank, reject the next tx, check fees, and round-trip all entries +# through actual shred generation and decoding, preserving transaction order/hash. +go test ./pkg/blockprod -run '^TestReadonlyPairBlockCapacityAndShredRoundTrip$' -count=1 + +# Whole-bank construction: one serial caller; pre-signed unique transactions. +GOMAXPROCS=8 go test ./pkg/blockprod -run '^$' \ + -bench '^BenchmarkReadonlyPair(FullBlock|PreparedFullBlock)$' -benchtime=3x -count=3 + +# More representative instruction workloads and smaller component microbenchmarks. +GOMAXPROCS=8 go test ./pkg/blockprod -run '^$' \ + -bench '^BenchmarkWorkingBank(Decoded|Prepared)?HotAccounts$' -benchtime=2s -count=3 +``` + +The whole-bank benchmark admits 48,622 transactions and includes execution, +account publication, entry batching/hash work and final entry flush. Signing and +bank setup are excluded. The prepared variant additionally does static +preparation before timing, modeling a queue ready before leadership. That work +is moved, not eliminated. Actual AccountsDB, signature verification, network, +consensus and the protocol deadline are outside this benchmark. The correctness +test's shred round-trip is also outside the timed benchmark. + +`txfixture.ReadonlyPairWire` provides the same ordered-pair construction as the +live experiment: 128×127 unique messages per payer/blockhash. Repeated ordinals +need a different payer or blockhash. The 200k test verifies uniqueness across +all messages and decodes/verifies representative signatures and phase boundaries. + +## Queue supply and completion reserve + +```toml +[validator] +tpu_max_buffered_transactions = 0 # default 131072 +block_completion_reserve_ms = 0 # default 75ms +``` + +The corresponding flags are `--tpu-max-buffered-transactions` and +`--leader-completion-reserve-ms`. The measured full-prefill trial used 262,144 +queue entries and a 60ms reserve. Those are opt-in tuning values; defaults stay +unchanged. A larger queue uses additional memory for owned wire, decoded and +prepared objects. A shorter reserve needs measured local finalization/broadcast +margin and does not change the protocol deadline. Shifting completion also +shifts later bank start times, so it does not add the same packing time to all +four blocks. + +The live results showed why total supply and timing matter: 120k transactions +cannot fill four approximately 48.6k blocks. An 80k refill competed with ongoing +work. Preloading 200k into a larger queue improved the observed four-block total, +while the first block still had less usable time and later banks awaited local +replay/adoption. These observations do not isolate a single CPU bottleneck. + +[Measured results, exact baselines and evidence](https://github.com/Overclock-Validator/mithril/blob/06ef067798c99947e8cc527450ad28430a9a7333/docs/results/leader-block-packing/2026-09-13/README.md) +include both the successful near-limit block and the still-underfilled four-slot +window. The archived live helper is historical experiment source with explicit +cluster/identity/path constants; the offline fixture is the portable reproduction. diff --git a/docs/out-of-order-entry-prefetch.md b/docs/out-of-order-entry-prefetch.md new file mode 100644 index 000000000..9645b6d3d --- /dev/null +++ b/docs/out-of-order-entry-prefetch.md @@ -0,0 +1,28 @@ +# Prepare complete entry batches despite earlier shred gaps + +Live tracing found five large external blocks whose final assembly-to-ready time was 20–39 ms. Most fallback verification work was already available more than 20 ms before full assembly. For one 33,760-transaction block, 6,506 transactions in later complete batches waited 44–116 ms for discovery behind an earlier missing shred. + +The old discovery cursor stopped at the first missing data index. The new bounded bitmap index discovers each complete DATA_COMPLETE range independently. Receiving a data shred or recovering one through FEC can release its containing batch and, when it supplies a boundary, the next batch. A batch still needs every data shred and its preceding boundary, unless it begins at index zero. Results are queued in discovery order and final assembly restores wire order using the existing exact range and byte-identity checks. + +The index uses about 24 KiB per retained slot and is allocated only with streaming preparation enabled. Successor/predecessor queries have bounded cost even with reverse or adversarial arrival order. Existing worker counts, verifier batching, retained-byte limits, cancellation, generation ownership, final block checks and signature validation remain unchanged. + +Regression coverage includes a delayed earlier shred, delayed preceding boundary, FEC-recovered boundary, disabled preparation, randomized arrival against a reference oracle, duplicate discovery, index word/group/slot boundaries, and the existing final-validation and cancellation tests. Native Turbine/replay race tests and Turbine vet passed. + +## Controlled Zen 5 benchmark + +AMD Ryzen 7 9700X, Go 1.26.4, Narya r51, GOMAXPROCS=8, two transaction verification workers, target batch size eight. 33,760 generated signed transactions arrive as complete component bursts across 200 ms. One data shred in a component three quarters through the block is withheld until after the footer. Both versions run with identical fixtures in baseline/candidate/candidate/baseline order, three iterations per case in each run, while the validator remains running. Ranges below are the two run medians, not a confidence interval. + +| Wire transaction size | Prior assembly → ready | New assembly → ready | +| --- | ---: | ---: | +| 228 bytes, delayed shred | 21.43–22.77 ms | 2.82–2.83 ms | +| 1,232 bytes, delayed shred | 35.46–36.34 ms | 7.04–8.08 ms | +| 228 bytes, ordered | 2.50–2.56 ms | 2.40–2.52 ms | +| 1,232 bytes, ordered | 6.91–7.67 ms | 6.87–7.89 ms | + +The delayed-shred case now verifies roughly 33.5–33.7k transactions before assembly, compared with 25.3–25.6k previously. The benchmark asserts every retained signature is verified exactly once and block metadata remains correct. It includes assembly, decoding, final validation and transaction verification; it excludes network I/O, replay execution and actual FAST-certificate inclusion. This modeled case is not an end-to-end validator speedup or a prediction of overall FAST percentage. + +Reproduce with `MITHRIL_SIGVERIFY_FLOW_BACKEND=r51 GOMAXPROCS=8 go test ./pkg/turbine -run '^$' -bench '^BenchmarkEntryPrefetchGapArrival$' -benchtime=3x` on each implementation, copying the same benchmark file to the baseline. + +Historical live trials and their limitations are in the +[archived evidence](streaming-preparation-evidence.md). Component results do not establish +a sustained FAST-inclusion improvement. diff --git a/docs/repair_sim.md b/docs/repair_sim.md new file mode 100644 index 000000000..b0240f015 --- /dev/null +++ b/docs/repair_sim.md @@ -0,0 +1,153 @@ +# Deterministic repair simulation + +`cmd/repair-sim` is a single-process harness for measuring the local path from +an incomplete slot to a block that is available to replay. It exists because +near-tip repair and deep catch-up optimize different outcomes: + +- near the tip, latency of the first replay-blocking slot matters; +- during catch-up, sustained useful data and completed slots per second matter. + +The first implementation deliberately stops before UDP and transaction +execution. It establishes a deterministic, correctness-checked baseline before +network realism or alternative scheduling policies are introduced. + +## What is real and what is simulated + +| Stage | Implementation | +| --- | --- | +| block-component serialization | production `turbine.MarshalBlockComponent` | +| 32+32 FEC generation and Merkle signing | production `turbine.Shredder` | +| missing-shred selection | production `SlotAssembler.RepairRequests` | +| packet parsing and Merkle/signature validation | production `ParseShred` and `ShredSignatureVerifier` (the receiver's per-root cache) | +| verified-shred insertion | production `ShredSpool` | +| threshold detection and Reed-Solomon recovery | production `SlotAssembler.AddShredFrom` | +| component decode and transaction-signature gate | production slot completion path | +| remote peer, latency, jitter, loss, duplication, bandwidth | deterministic in-process simulator | +| replay notification | block emission is recorded as “offered to replay” | +| transaction execution | not run in this version | + +Synthetic entries are valid Alpenglow entry-batch components but contain no +transactions. This isolates shred/FEC/repair/storage costs; it is not a replay +execution benchmark. + +## Scenarios + +### Near tip + +`near-loss` alternates two useful patterns across FEC sets: + +- 30 data + 1 coding shred: one repair response crosses the threshold and + reconstructs the other missing data shred; +- 31 data shreds: the one missing data shred must be fetched directly. + +Selected omitted shreds can also arrive through the simulated live path while +a repair response is outstanding. This measures cancellation/late-response +behavior without changing production scheduling. + +Disabling repair stops request scheduling but still delivers natural late live +shreds. The run ends when all slots complete or those live arrivals are +exhausted; incomplete slots are reported without a repair-stall error. + +### Deep catch-up + +`mixed` begins every FEC set with 16 data + 15 coding shreds. One fetched data +shred crosses the threshold and reconstructs the remaining 15. + +`sparse` begins every FEC set with two data shreds and no coding layout. The +ordinary repair interface serves data shreds only, so nearly all missing data +must arrive over the simulated network. Comparing `mixed` with `sparse` +quantifies the network work avoided by already-held coding shreds. + +## Commands + +```sh +go test ./pkg/turbine/repairsim + +go run ./cmd/repair-sim \ + -scenario=near-tip \ + -slots=200 \ + -fec-sets=4 \ + -seed=1 \ + -cpu-label='Ryzen 7 9700X' \ + -output=/tmp/repair-near.json + +go run ./cmd/repair-sim \ + -scenario=deep-catchup \ + -availability=mixed \ + -slots=1000 \ + -fec-sets=4 \ + -seed=1 \ + -repair-latency=20ms \ + -repair-bandwidth=104857600 \ + -output=/tmp/repair-deep-mixed.json + +go run ./cmd/repair-sim \ + -scenario=deep-catchup \ + -availability=sparse \ + -slots=1000 \ + -fec-sets=4 \ + -seed=1 \ + -repair-latency=20ms \ + -repair-bandwidth=104857600 \ + -output=/tmp/repair-deep-sparse.json + +go test ./pkg/turbine/repairsim \ + -run '^$' -bench '^BenchmarkScenarios$' -benchmem -count=5 +``` + +Logical trace timestamps are deterministic. `wall_elapsed_ns`, allocations, +and `stage_cpu_ns` are actual local measurements and therefore are not expected +to be byte-identical across runs. + +## Correctness gates + +The current tests require: + +- exact requested FEC-set counts from authentic generated shreds; +- Merkle/signature validity for every canonical packet; +- no completed slot when loss is present and repair is disabled; +- complete, canonical entry streams after threshold recovery; +- a complete `ShredSpool` journal record before replay admission; +- rejection and retry of a corrupted repair response; +- deterministic logical traces for identical seeds; +- late repair responses to leave a completed block unchanged. + +The Turbine package also retains focused byte-for-byte Reed-Solomon recovery +tests. The simulator verifies the stronger end-to-end consequence: recovered +shreds must decode to the exact canonical entry sequence and pass the normal +completion gates. + +## Generator compatibility finding + +Building this harness exposed a multi-FEC component bug: the local generator +set `DATA_COMPLETE_SHRED` at every FEC boundary. The decoder correctly treats +that flag as the end of one serialized component, so a component spanning more +than one FEC set was truncated and failed to decode. The generator now follows +Agave's ordering: construct every FEC set, then mark only the final data shred +of the component complete (or last-in-slot). A regression test round-trips one +1,300-entry component across multiple FEC sets. + +## Future mode selection + +No production mode switch is added here. A later policy experiment should use +both replay distance and observed Turbine usefulness, with hysteresis: + +- stay in near-tip mode while the replay gap is small and Turbine supplies a + high fraction of useful shreds before repair deadlines; +- enter catch-up mode only when the replay-blocking gap is sustained and live + Turbine delivery is insufficient to approach FEC thresholds; +- return to near-tip mode only after both the gap and repair backlog fall below + lower thresholds. + +That signal is preferable to slot distance alone: a node may be numerically +close to the tip while receiving too few live shreds, or far behind while its +local spool already holds most FEC thresholds. + +## Next steps + +1. Add shallow catch-up and whole-block-pressure configurations. +2. Add a loopback-UDP transport without replacing the deterministic mode. +3. Expose internal FEC start/finish timing through opt-in instrumentation. +4. Run replay execution against a reusable synthetic bank fixture. +5. Compare ordinary requests with explicit test-only threshold-acquisition and + earliest-blocked-slot policies. diff --git a/docs/shred-spool-completion.md b/docs/shred-spool-completion.md new file mode 100644 index 000000000..b726265f3 --- /dev/null +++ b/docs/shred-spool-completion.md @@ -0,0 +1,85 @@ +# Shred-spool completion publication + +`MarkComplete` runs on the block-delivery path. Previously it wrote a 16-byte +`complete.idx` record while holding the spool mutex. This write uses no fsync, +but ordinary filesystem writes can still wait. Earlier live observations included +32–40 ms completion-to-delivery delays; one separate detailed trace localized a +37.8 ms pause to the journal write on an already-adopted own-leader block. That +own-leader example did not delay its vote. Do not equate all completion-to-delivery +stalls with journal I/O without the finer trace. + +Completion publication now updates the in-memory map and makes a nonblocking +submission to one writer with a 256-record queue. The writer never takes the +spool mutex. A full queue drops only the persistent completion hint; in-memory +completeness remains available. This bounds pending memory and prevents storage +backpressure from directly reaching completion publication. No worker-count or +validator configuration changes are required. + +## Recovery and ownership contract + +The spool is a disposable verified-shred cache, not vote history, account state, +or a durable checkpoint. Completion hints can be lost on an unexpected stop or +queue overflow. Recovery then reassembles/re-repairs; a hint never replaces the +assembler's coverage and block validation. The packet checksums and journal/file +formats are unchanged. There is no new fsync or power-loss durability guarantee. + +Deletion and corrupt-tail truncation are different: they must not race an older +queued completion. Their tombstone goes through the same ordered writer, and the +caller waits for it **before changing the slot file**. Thus an older queued hint +cannot be written after the tombstone and resurrect completeness for a replacement +partial file. These uncommon operations can still wait for storage while holding +the spool mutex; this change does not remove every source of spool contention. + +After a short/error write, the worker stops appending hints and truncates the +journal to zero. This avoids appending behind a partial record and removes old +completion hints before an invalidation is acknowledged. If truncation also fails, +the invalidation fails and the slot-file mutation is refused; a later attempt can +retry. Losing all cached completion hints is an acceptable repair-cost fallback. + +`Close` excludes further mutations, flushes packet buffers, retries current +completion hints if the queue overflowed, and drains/closes the journal worker +before returning. The next opener therefore preserves the existing clean-handoff +contract when storage succeeds. A stuck disk can still delay shutdown. The worker +must not outlive ownership of the spool directory. No voting-resume, signing-bound, +checkpoint-coverage or consensus-safety rule changes. + +## Validation and measurements + +Tests block the writer and overflow the queue while asserting that completions +remain available, then verify clean-close recovery. A replacement-file test keeps +the old completion write blocked and verifies that replacement cannot proceed +before its tombstone. Fault tests inject partial writes and failed truncation, +verify refusal to mutate the slot, then retry and check that no stale hint returns +on restart. Existing checksum/torn-tail, retention, handoff, receiver shutdown and +invalid-block tests remain covered. + +Native full turbine/blockstream race suites, vet and the combined validator build +passed on the Ryzen 9700X (Zen 5), Go 1.26.4. The test process used GOMAXPROCS=2, +Nice=19 and a 200% CPU quota on the active validator host. + +Run the identical `BenchmarkShredSpoolMarkComplete` file on both source revisions. +Each sample opens a fresh spool, appends a packet outside the timer, times one +completion, then closes/drains outside the timer. Thus no already-complete dedupe +or queue-overflow drop is measured. Three runs of 300 iterations: + +| Component | Before | Candidate | +|---|---|---| +| Median run p50 | 2.805 µs | 0.170 µs | +| Median run p99 | 6.201 µs | 0.581 µs | + +This measures ordinary storage, not injected tail latency, total CPU work, replay +or FAST inclusion. Disk work moves to the worker; it does not disappear. The +baseline is the exact previously deployed combined validator, SHA256 +`a58366704680154628ff0a4c6b4027a3e5b79f1909eb39bffeec326c008f0b68`, not the full +branch versus alpenglow-dev. + +## Limits + +The blocked-writer regression establishes isolation of completion publication. +It does not eliminate all spool I/O: ordered invalidations, slot-file operations +and shutdown can still wait for storage. Historical live trials did not establish +an overall large-block p99 improvement and included remaining verification tails. +Keep those limits separate from the component benchmark above. + +[Historical measurements and source](spool-completion-journal-evidence.md) retain +the original deployment comparison and its trace/probe qualifications. diff --git a/docs/shred_retention_performance.md b/docs/shred_retention_performance.md new file mode 100644 index 000000000..ebcb9af82 --- /dev/null +++ b/docs/shred_retention_performance.md @@ -0,0 +1,44 @@ +# Shred retention sweep scheduling + +`SlotAssembler` previously scanned its incomplete/completed slots, block-ID hints, +rejected IDs, partial observations, and priority repair state on every incoming +shred, including duplicates and completed-slot packets. The new schedule skips +age sweeps when neither the observed edge nor the retention floor nor relevant +retained state has changed. The original sweep algorithm and age limits remain. + +Mutations invalidate the cached sweep after adding old identity hints or partial +observations, resetting a generation, or terminating completion. Completion +success, error, cancellation and abort all release protected parent-ID state. +Repair-floor advancement, reduction and clearing are checked on the next packet. +The hard incomplete-slot capacity check remains unconditional on every packet, +including catch-up insertion at an unchanged edge. Eviction preference and +protection for completing generations and the repair head are unchanged. + +Regression coverage includes changing the repair floor without advancing the +edge, every terminal completion outcome, newly added old identity hints, and +capacity overflow at a fixed edge. Existing completion, cancellation, generation, +FEC and repair tests also run as part of the full Turbine suite. + +## Isolated benchmark + +`BenchmarkRetentionRepeatedCompletedShred` drives the public `AddShred` path +with a completed-slot packet and 513 retained entries in each of four metadata +maps. It measures the repeated scan/rejection case, not full packet decoding, +authentication, FEC, replay, or a whole-validator speedup. Five 500 ms runs: + +| Host | Baseline median | Candidate median | Allocation | +| --- | ---: | ---: | ---: | +| Apple M4 Pro | 11,617 ns/packet | 7.718 ns/packet | 0 on both | +| Ryzen 7 9700X, GOMAXPROCS=2 | 11,443 ns/packet | 12.99 ns/packet | 0 on both | + +Baseline and candidate use the same fixture; Go's source overlay selects the old +assembler for baseline runs without changing other source. This fixture shows the +avoided work, not a prediction for a validator's actual map population. + +Validation passed locally and natively: race suites for Turbine, replay, +consensus and node; Turbine vet; complete validator build. The live integration +applies this patch over the exact deployed FEC/peer-isolation/status-expiry source. + +Historical live trials and their limitations are in the +[archived evidence](streaming-preparation-evidence.md). Component results do not establish +a sustained FAST-inclusion improvement. diff --git a/docs/spool-completion-journal-evidence.md b/docs/spool-completion-journal-evidence.md new file mode 100644 index 000000000..d2b40d986 --- /dev/null +++ b/docs/spool-completion-journal-evidence.md @@ -0,0 +1,14 @@ +# Spool Completion Journal: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/f0b72ab239efbbd4811498d73b36ba73eb6192e1) +(tag `review-evidence-20260916-spool-completion-journal`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/f0b72ab239efbbd4811498d73b36ba73eb6192e1/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/docs/streaming-preparation-evidence.md b/docs/streaming-preparation-evidence.md new file mode 100644 index 000000000..53e444472 --- /dev/null +++ b/docs/streaming-preparation-evidence.md @@ -0,0 +1,14 @@ +# Streaming Preparation: benchmark evidence + +The maintained subsystem documentation and reusable Go benchmarks describe the +implementation and reproduction method. Historical raw results and session +notes are retained at [the tested source snapshot](https://github.com/Overclock-Validator/mithril/tree/1c1171d3661d0404b013a9bf9391e23eb660706e) +(tag `review-evidence-20260916-streaming-preparation`). They are omitted from this proposed merge. + +[Historical result files](https://github.com/Overclock-Validator/mithril/tree/1c1171d3661d0404b013a9bf9391e23eb660706e/docs/results) + +Measurements retain their original baselines. Rebasing onto PR #278 does not +turn an intermediate-version benchmark into a comparison with the new base. +Component timings and short live observations do not establish sustained FAST +inclusion gains. The final review description records validation of the rebased +source separately from historical benchmark results. diff --git a/docs/streaming_message_identities.md b/docs/streaming_message_identities.md new file mode 100644 index 000000000..d852b22d4 --- /dev/null +++ b/docs/streaming_message_identities.md @@ -0,0 +1,106 @@ +# Prepare transaction message identities during shred arrival + +The live Zen 5 admission trace found ~12.45 ms of message serialization, +hashing and identity-cache preparation after assembly on large blocks, followed +by ~1.66 ms of same-block duplicate-plan work. Most blocks reached this stage +while replay was already waiting. + +Turbine now asks signature-verification workers to retain a message identity +derived from the exact canonical bytes they already serialize for verification. +The existing two workers process available groups without waiting for more +transactions. TPU callers of the ordinary verifier do not compute these extra +identities. + +Each opaque result binds a successful signature verdict to the transaction +pointer, message version and recent blockhash. Completion joins requests and +imports only identities covering the final ordered transaction slice. Canceled +requests are reverified; discarded UpdateParent prefixes are excluded. Results +are caller-owned and cannot refer to reusable verifier scratch. The block cache +owns its imported storage and remains nonserialized. The existing requirement +that signed message contents remain immutable still applies; arbitrary in-place +message edits require invalidation, as before. + +Same-block duplicate rejection and mutable ancestor/status checks remain in +place. This change does not cache the duplicate-check plan or alter epoch +processing, vote persistence, worker counts, scheduling deadlines, or packing. + +## Zen 5 comparison + +Baseline: the currently deployed shred-retention source, including its existing +FEC and peer-isolation integrations. Candidate: that same source plus streaming +identities. Both binaries use the identical new benchmark harness. + +`BenchmarkEntryMessageIdentityArrival` feeds real generated data shreds through +the assembler, entry prefetch and signature-verification pipeline, then includes +the admission-time identity lookup. Each block has 33,760 single-signature +transactions, either 228 or 1,232 bytes on the wire. The tip model schedules +component arrivals across 200 ms; catchup offers every shred immediately. +Fixture construction, packet parsing and shred-signature authentication are +outside the timer. There is no network loss, transaction execution, PoH/reward +processing or final whole-block duplicate map in this benchmark. + +AMD Ryzen 7 9700X, Narya `r51` AVX-512 backend, two signature workers, target eight +signature lanes, GOMAXPROCS=16. Five iterations per scenario per run, two runs +per variant in baseline/candidate/candidate/baseline order. Values below are the +median of the two run medians, not a percentile computed over all ten samples. +The live validator and load services continued running, so host contention can +affect the results. + +| Arrival model | Transaction size | Last shred → identities available, baseline | Candidate | +| --- | ---: | ---: | ---: | +| Over 200 ms | 228 bytes | 13.62 ms | 2.66 ms | +| Over 200 ms | 1,232 bytes | 73.72 ms | 7.81 ms | +| Catchup | 228 bytes | 92.90 ms | 88.40 ms | +| Catchup | 1,232 bytes | 154.75 ms | 118.20 ms | + +The table includes completion work; it does not merely move that work out of +the admission timer. Admission's identity lookup alone falls from ~12.1 to +0.24 ms for 228-byte tip transactions and ~66.9 to 0.26 ms for maximum-size tip +transactions. The block-wide duplicate check remains additional work. + +CPU time per tip block was 189.75→197.20 ms for the small fixture (~4% higher) +and 316.40→304.80 ms for the maximum-size fixture (~4% lower). The intended gain +is less work after arrival, not a blanket claim of lower CPU. Maximum-size +fixtures allocate roughly 37 MB fewer bytes per block by avoiding a second +message serialization; retaining opaque identities also has a memory cost. + +The initial runs without an explicit backend used the library's unconfigured +default and are excluded from these deployment-relevant results. + +## Validation and current limits + +Local affected-package tests, race tests for txverify/block/turbine/replay, Go +vet, and a full validator build passed. Tests cover legacy/v0/v1 canonical +identities, failed signatures, scratch reuse, pointer/order/blockhash mismatch, +storage ownership, JSON round-trips, early preparation and mixed canceled-batch +fallback. Existing turbine tests cover invalid and discarded prefixes and +completion cancellation. + +Historical deployment observations are retained in the +[archived evidence](streaming-preparation-evidence.md). Their unequal workloads +and epoch boundary do not establish an isolated FAST-score improvement. + +## Recovery from inconsistent prefetch metadata + +Missing/oversized retained ranges, partial identities and identity-binding +mismatches now log a warning and fall back to full signature verification of the +final block. Bounds are checked before slicing the final transaction array. Old +readers are joined first, including on cancellation. Valid final transactions +can therefore recover from an optimization bookkeeping fault; a successful +cached verdict for different bytes cannot authorize the final transaction. +Normal cached signature failures remain errors, as do failed re-verification, +cancellation and a closed verifier. No signatures or duplicate checks are skipped. + +Regression tests cover valid and invalid final blocks for each metadata fault, +cache identities matching the final transaction order, canceled-reader ownership +and verifier failure. The ordinary path retains exact-range/byte checks and +verified identity reuse. Full re-verification is exceptional and costs additional +work; this change is a correctness/availability fix, not a throughput claim. + +The `[sigverify]` starter configuration and its test moved here from the voting +branch, because this branch reads those keys and supports the legacy backend key. +The unrelated explicit `tuning.use_pool` template setting is omitted: enabling +pooling by default and fixing retained vote ownership belong to the runtime +branch. In the combined build, its configuration defaults still enable pooling. +The default remains two Turbine verification workers (bounded by GOMAXPROCS). +Many-core catch-up tuning remains a separate measurement question. diff --git a/docs/transaction_sigverify_streaming.md b/docs/transaction_sigverify_streaming.md new file mode 100644 index 000000000..42e4bf311 --- /dev/null +++ b/docs/transaction_sigverify_streaming.md @@ -0,0 +1,159 @@ +# Transaction signature verification during shred arrival + +Turbine verifies transaction signatures as complete entry batches arrive. The +default shared transaction pool has `min(2, GOMAXPROCS)` workers and targets eight signature +lanes. A ready batch containing four transactions runs immediately; there is no +timer or minimum occupancy requirement. The same policy handles live reception +and repair catch-up without a mode transition or a 200 ms batching delay. + +## Order of work + +1. The receiver authenticates the shred and performs existing assembly/FEC + recovery. The packet reader advances a contiguous data-shred frontier and + attempts a nonblocking background enqueue. Assembly retains an authenticated + root and a snapshot of its source shred for each FEC set when available. +2. Two background preparation workers copy and decode complete DATA_COMPLETE + entry batches. A batch may span several FEC sets. They submit its immutable + transactions to the shared signature pool while later shreds arrive. +3. Signature groups contain available transactions up to the configured lane + target. Transactions with multiple signatures remain indivisible. Large + already-decoded requests bundle four vector groups per dispatch job (normally + 32 one-signature transactions). Requests smaller than + `2 * workers * batch_target * 4` transactions keep one group per job; the + default threshold is 128 ready transactions. A rolling per-request window + refills when any job finishes, without a wave barrier or batching timer. +4. Once the full slot is assembled, completion compares shred slices directly + against the cached component bytes, including padding. Cache hits avoid a + second component buffer; misses allocate a fresh buffer at its exact size. + Complete contiguous slots supply shred order directly, avoiding two sorts. + Completion processes all Alpenglow markers and FEC roots and constructs the + final ordered block. It submits any transactions not already covered and joins + signature work for the retained batches before marking the block verified. +5. The existing replay pipeline receives the verified block. This change does + not execute transactions before full-slot admission, alter execution batching, + or move parent-dependent validation ahead of its required state. + +The 200 ms slot interval provides an opportunity to overlap work. It is not a +mandatory local wait: replay can continue as soon as the block is available and +its checks complete. If all shreds arrive in a burst during catch-up, full-slot +completion uses the same efficient groups with whatever early work was able to +start. Four workers can improve catch-up latency but occupy more cores at once. + +## Bounds and correctness + +- At most eight slot generations hold early preparation reservations, with a + combined 64 MiB budget for raw component bytes and a 1 MiB per-component limit. + Decoded transaction objects add heap overhead. Saturation skips optional early + work; normal full-slot verification still covers every retained transaction. +- One queued/active preparation token per generation coalesces packet arrivals. + Verifier requests and each request's outstanding jobs are also bounded. + Request admission can wait behind existing requests; this is not a strict + replay-head priority scheduler. +- No transaction decoding or verifier admission occurs under the assembler mutex or on the + packet reader. A gap prevents early component decoding until recovery or + arrival closes it. Duplicate shreds do not create duplicate requests. +- Cached results belong to one generation, shred range and exact byte sequence. + Reset/eviction cancels that generation. Reservations remain charged until + admitted readers have relinquished their transaction buffers. +- A FEC root cache retains at most one source snapshot per FEC state, in addition + to the entry-prefetch budget. Completion preserves the deterministic choice of + the lowest-index non-recovered data proof, then lowest coding position. Cached + roots require the same source, parsed root inputs, and exact payload bytes; + mismatches, unauthenticated callers, and spool hydration recompute the root. +- `UpdateParent` can discard an optimistic prefix. Parse and marker checks still + cover that prefix, while its transaction signature verdict is discarded along + with its transactions. Retained signatures must all pass before replay. +- Cancellation is not an invalid-signature verdict. A retry on the same slot + generation verifies transactions again if an earlier request was canceled. + An admitted job finishes its first vector group; cancellation can skip later + groups in that job. The request joins all admitted jobs before releasing input. + +## Configuration and observability + +```toml +[sigverify] +backend = "auto" +workers = 0 # min(2, GOMAXPROCS) +batch_target = 8 # 4 or 8; short groups never wait to fill +disable_shred_overlap = false +``` + +Equivalent CLI flags are `--sigverify-workers`, `--sigverify-batch-target` and +`--sigverify-disable-shred-overlap`. These settings apply to Turbine transaction +signatures; TPU, shred signatures, consensus BLS and replay's fallback verifier +retain their existing configuration. + +Use `TurbineFullToReady` to measure residual wall time after the slot becomes +complete. `TurbineEarlyVerifiedTransactions` counts retained transactions whose +verification finished before full assembly. `TurbineTransactionSigverify` now +measures completion's outstanding-signature join/fallback work. The remaining +wait for an already claimed background preparation job, including +any outstanding admission delay, appears in `TurbineEarlyPreparationWait`, +separately from active completion decoding. It does not sum every background +admission wait. Early parse and signature durations are component sums observed +at completion; +a discarded prefix still being verified is not included. These durations +overlap reception and each other and must not be added as sequential stages +or interpreted as CPU time. + +## Benchmark scope + +`BenchmarkTransactionVerificationFlow` compares two/four workers and targets +four/eight using catch-up, synthetic 200 ms component arrivals, and sparse +four/seven/eight-transaction components. It supports captured public transaction +fixtures and generated, distinct valid transactions of exactly 228 or 1,232 wire +bytes. Decoding and fixture construction are outside those pool measurements. + +The execution contention probe runs the real transfer load/execute benchmark +alongside signature work on the same eight physical cores. Its separate +processes measure hardware/OS contention, excluding shared Go scheduler/heap +effects, full-block dependency planning, commit and live network timing. Pool +throughput alone is insufficient evidence of end-to-end replay improvement. + +Measured Zen 5 results, raw logs, and validation details are in the +[September 12 benchmark report](https://github.com/Overclock-Validator/mithril/blob/1c1171d3661d0404b013a9bf9391e23eb660706e/docs/results/sigverify-streaming/2026-09-12-zen5/README.md). +The subsequent [direct cache-comparison report](https://github.com/Overclock-Validator/mithril/blob/1c1171d3661d0404b013a9bf9391e23eb660706e/docs/results/sigverify-direct-cache/2026-09-12-zen5/README.md) +isolates the removal of redundant component-buffer construction at completion. +The [completion follow-up report](https://github.com/Overclock-Validator/mithril/blob/1c1171d3661d0404b013a9bf9391e23eb660706e/docs/results/completion-followup/2026-09-12-zen5/README.md) +measures direct ordering, authenticated-root reuse, and the four-vector job policy. + +The [standalone PR review](https://github.com/Overclock-Validator/mithril/blob/1c1171d3661d0404b013a9bf9391e23eb660706e/docs/results/streaming-pr-review/2026-09-13/README.md) +records extraction onto current `alpenglow-dev`, the small shared component-boundary +prerequisite, final allocation improvement, and the scope of the live trial. + +## Worker default compatibility + +The automatic transaction-verifier default changes from `(GOMAXPROCS + 1) / 2` workers to +`min(2, GOMAXPROCS)`, including when shred overlap is disabled. This favors spare +CPU capacity for execution and other verification at the tip. It is not a claim +of maximum catch-up throughput on every core count or backend. Set +`--sigverify-workers N` or `[sigverify] workers = N` explicitly when tuning a +larger machine; disabling overlap alone does not restore the previous worker +count. Existing two/four-worker contention measurements are in the September 12 +report above. No 32-core comparison was performed. + +## Reserved admission for completion + +Decoded prefetch components now use a separate admission class. The existing total request limit remains `2 * workers`; at most `2 * workers - 1` requests may prefetch. Thus the default two-worker pool keeps four total permits, with at most three occupied by prefetch. Waiting completion/full-block recovery requests win the next free permit over prefetch. Their admission, cancellation and close registration share the verifier mutex; notification channels are allocated only when callers must wait. + +No worker or job queue is added, and verification, vector width, job grouping and per-request rolling windows are unchanged. Accepted jobs finish normally and are joined before transaction memory can be reused. No signature checks are skipped. Prefetch may wait while completion callers remain queued and resumes when that backlog drains. This is completion-class priority, not exact replay-head priority: future-slot completions also qualify, and already admitted prefetch work is not promoted or preempted. Checkpoint, persistence and voting-recovery contracts are unchanged. + +### Saturation benchmark + +`BenchmarkVerifierCompletionReservation` verifies four already-ready prefetch components (256 or 4,096 signed 228-byte transactions each) plus a 32-transaction completion request. All requests are joined; total-work timing includes all four components. Two verifier workers, eight signature lanes, four vector groups, GOMAXPROCS=8, Narya r51, Ryzen 9700X / Go1.26.4. Three runs of 100 iterations each, Nice19 and a 200% CPU quota on the shared validator host. Test intervals are excluded from live FAST comparisons. + +The before comparison uses the previously deployed source (status-validation combined build, SHA256 `2c81fc403e8a6eca73b87ede51890041347e1af0e3c63df005fcfeb26448e872`) with only the benchmark added via a Go test overlay. The candidate also measures the shared-class control to distinguish policy from incidental overhead. These are incremental admission results, not the whole PR versus alpenglow-dev. + +For four 4,096-transaction components, medians of the three per-run statistics were: + +| Measurement | Previous deployment | Reserved admission | +|---|---:|---:| +| Completion admission p50 | 37.19 ms | 0.000742 ms | +| Completion admission p99 | 45.78 ms | 0.004599 ms | +| Completion finished p99 | 47.31 ms | 1.488 ms | +| All work finished p50 | 39.43 ms | 39.11 ms | +| All work finished p99 | 49.09 ms | 50.65 ms | + +Completion-finished p99 ranged 46.43–54.52 ms before and 1.477–1.719 ms after. Admission p99 ranged 44.27–53.76 ms before and 0.003206–0.06401 ms after. Every iteration reached its intended request occupancy. For 256-transaction components, completion-finished p99 medians were 3.215→1.165 ms. Shared-host scheduling introduces variation; reserving admission does not remove queued-job or CPU delays, and these 100-sample tails are not a live p99/FAST claim. Total-work throughput was roughly unchanged; no total-work tail improvement is claimed. + +Original run artifacts are retained in the [evidence archive](streaming-preparation-evidence.md). diff --git a/docs/turbine-relay-buffers.md b/docs/turbine-relay-buffers.md new file mode 100644 index 000000000..3ee75f3f0 --- /dev/null +++ b/docs/turbine-relay-buffers.md @@ -0,0 +1,60 @@ +# Turbine relay buffer ownership + +Each retransmit worker reuses a private peer-result slice. Weighted shuffle, +tree placement, address filtering and fanout are unchanged. The exported +`RetransmitPeers` API still returns independently allocated result storage. +Workers clear their entire scratch slice after each send so it cannot retain +addresses from an old cluster snapshot. + +`SubmitFrom` copies a packet into exclusively owned storage before returning +to its caller. Canonical packets use a fixed-size, GC-reclaimable `sync.Pool`; +oversized inputs retain the independent allocation path. Queue admission +transfers ownership to the worker. The worker returns storage only after all +synchronous sends and retries finish, including error and no-peer paths. +Queue rejection returns storage immediately. Shutdown closes admission under +a short lock, joins workers, then drains remaining copies. The channel stays +open for concurrent submitters, which cannot enqueue after admission closes. + +`packetBatchSender.Send` borrows both packet bytes and peer addresses only +until return, even on partial sends or errors. Implementations that retain +either must copy them. A pooled packet must never escape this lifetime. +The admission lock is not held during authentication, routing or socket I/O. + +## Measurement + +`BenchmarkRetransmitPipeline` measures deduplication, packet copying, queue +handoff, weighted routing and dispatch to a mock sender. Run with: + +```sh +GOMAXPROCS=2 go test ./pkg/turbine -run '^$' \ + -bench '^BenchmarkRetransmitPipeline$' -benchmem -benchtime=20000x -count=3 +``` + +On a Ryzen 7 9700X (Zen 5), Go 1.26.4, one producer and one relay worker, +the incremental comparison against the relay implementation at +`c40ac9e8ca8aa09a9f2a8759a231199b0f90346e` was: + +| Gossip contacts | Before ns/shred | After ns/shred | Before B/shred | After B/shred | Allocations before → after | +|---|---:|---:|---:|---:|---:| +| 90 | 3,906 | 3,761 | 3,784 | ~729 | 8 → 6 | +| 512 | 23,789 | 22,056 | 3,784 | ~729 | 8 → 6 | + +Times are medians of six samples per version, in baseline/candidate/candidate/ +baseline order, three samples per round. Both versions use the same benchmark +and combined validator source; only the two relay production files change. +Each iteration submits a distinct 1,203-byte data shred with cached topology; +the benchmark checks that none are dropped and waits for workers to finish. +The fixture includes staked and zero-stake peers. Contact count does not mean +that every shred has that many recipients: tree position determines forwarding. + +Allocation fell about 81%. The 4–7% median timing improvement is modest and +noisy on the shared validator host; one candidate sample was slower than every +baseline sample in its case. These are amortized in-memory pipeline times, +not network delivery latency or live FAST-score gains. Socket syscalls, parent +authentication and retransmitter signing are excluded from this fixture. + +Routing tests compare against a full weighted permutation, including both +shuffle modes and missing/unroutable contacts. Ownership tests exercise caller +buffer reuse, queue pressure, send retries, concurrent routing and shutdown. +The full Turbine race suite passed locally and natively; native vet and the +combined validator build passed. Historical run artifacts are linked from [the evidence archive](streaming-preparation-evidence.md). diff --git a/go.mod b/go.mod index 4607b7603..2463c0b17 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.26.4 replace github.com/gagliardetto/binary => github.com/palmerlao/binary v0.0.0-20250617062159-3054b4d33aed require ( - github.com/Overclock-Validator/narya-ed25519 v0.0.0-20260726222623-da0d045dae9d + github.com/Overclock-Validator/narya-ed25519 v0.0.0-20260730051143-c265ee966713 github.com/cespare/xxhash/v2 v2.3.0 github.com/charmbracelet/bubbles v0.21.1-0.20250623103423-23b8fd6302d7 github.com/charmbracelet/bubbletea v1.3.10 diff --git a/go.sum b/go.sum index 069571ee5..086dc43c2 100644 --- a/go.sum +++ b/go.sum @@ -12,8 +12,8 @@ github.com/Overclock-Validator/crypto v0.0.0-20250307094320-aaf52fac5261 h1:Y715 github.com/Overclock-Validator/crypto v0.0.0-20250307094320-aaf52fac5261/go.mod h1:ZhRHOaVg8I1gg0VK4wmqOQPnlgPgKFT9McZ+TCW/hBA= github.com/Overclock-Validator/gnark-crypto v0.0.0-20250309203346-2a67ed08a105 h1:mP6FWHZ8ddcmbE8UTrVVI2Mi2c24aqX/8p12Vn6zokQ= github.com/Overclock-Validator/gnark-crypto v0.0.0-20250309203346-2a67ed08a105/go.mod h1:Poczuq3dbt+CwyTKgOjGaEwJOMP7YxQobF7QhgNcguk= -github.com/Overclock-Validator/narya-ed25519 v0.0.0-20260726222623-da0d045dae9d h1:ipaL+9MHKeI8QIfWneId0VLa+STLRM1e6MnvZhQyhPU= -github.com/Overclock-Validator/narya-ed25519 v0.0.0-20260726222623-da0d045dae9d/go.mod h1:B7/xqV/5NtGJa8OlZAa9TRMHgeIE+VEJNiPzMP4FrIg= +github.com/Overclock-Validator/narya-ed25519 v0.0.0-20260730051143-c265ee966713 h1:nRAD+snanlR/sX4W5rkrxr6+sYBZ3Hl2xZQ6HCpM0Hc= +github.com/Overclock-Validator/narya-ed25519 v0.0.0-20260730051143-c265ee966713/go.mod h1:B7/xqV/5NtGJa8OlZAa9TRMHgeIE+VEJNiPzMP4FrIg= github.com/Overclock-Validator/solana-snapshot-finder-go v0.0.0-20260223201452-d8363b514fc0 h1:elgavEQb8l7Zn3gS3Y+2/98PlUylOWdlM3V1VumQ7mA= github.com/Overclock-Validator/solana-snapshot-finder-go v0.0.0-20260223201452-d8363b514fc0/go.mod h1:XbqbvMA2NKeosY0w3WdBOpCg2eYJesBjfE4cNt9HSE8= github.com/Overclock-Validator/wide v0.0.0-20250221123529-f80959d02044 h1:ph9gnWIY116AWT/iCfXoPe9/cn2aWx2uJBuLdf/LyEE= diff --git a/pkg/accounts/mem_accounts.go b/pkg/accounts/mem_accounts.go index 4158be553..630ffb420 100644 --- a/pkg/accounts/mem_accounts.go +++ b/pkg/accounts/mem_accounts.go @@ -1,7 +1,6 @@ package accounts import ( - "fmt" "sync" "github.com/Overclock-Validator/mithril/pkg/base58" @@ -13,6 +12,16 @@ type MemAccounts struct { mu *sync.RWMutex } +// A miss is an ordinary step when falling back to parent accounts. Defer the +// diagnostic encoding until it is needed, and copy the key so callers can reuse it. +type missingMemAccountError struct { + key [32]byte +} + +func (e *missingMemAccountError) Error() string { + return "no such account " + base58.Encode(e.key[:]) + " found" +} + func NewMemAccounts() MemAccounts { return MemAccounts{ Map: make(map[[32]byte]*Account), @@ -32,7 +41,7 @@ func (m MemAccounts) GetAccount(pubkey *[32]byte) (*Account, error) { defer m.mu.RUnlock() acct, ok := m.Map[*pubkey] if !ok { - return nil, fmt.Errorf("no such account %s found", base58.Encode(pubkey[:])) + return nil, &missingMemAccountError{key: *pubkey} } return acct, nil } @@ -40,7 +49,7 @@ func (m MemAccounts) GetAccount(pubkey *[32]byte) (*Account, error) { func (m MemAccounts) GetAccountWithoutLock(pubkey solana.PublicKey) (*Account, error) { acct, ok := m.Map[pubkey] if !ok { - return nil, fmt.Errorf("no such account %s found", base58.Encode(pubkey[:])) + return nil, &missingMemAccountError{key: pubkey} } return acct, nil } diff --git a/pkg/accounts/mem_accounts_test.go b/pkg/accounts/mem_accounts_test.go index bd55cb218..4a614dd9f 100644 --- a/pkg/accounts/mem_accounts_test.go +++ b/pkg/accounts/mem_accounts_test.go @@ -3,8 +3,23 @@ package accounts import ( "testing" "time" + + "github.com/gagliardetto/solana-go" ) +func TestMemAccountMissingErrorRetainsLookupKey(t *testing.T) { + mem := NewMemAccounts() + key := [32]byte{} + _, lockedErr := mem.GetAccount(&key) + _, unlockedErr := mem.GetAccountWithoutLock(solana.PublicKey(key)) + key[0] = 99 // Lookup callers may reuse their key storage before reporting an error. + for _, err := range []error{lockedErr, unlockedErr} { + if err == nil || err.Error() != "no such account 11111111111111111111111111111111 found" { + t.Fatalf("missing error lost its original key: %v", err) + } + } +} + func TestMemAccountsReadsAreConcurrent(t *testing.T) { mem := NewMemAccounts() var key [32]byte diff --git a/pkg/block/block.go b/pkg/block/block.go index 23bfd1f03..459fd68d5 100644 --- a/pkg/block/block.go +++ b/pkg/block/block.go @@ -14,15 +14,30 @@ import ( "github.com/gagliardetto/solana-go/rpc" ) -// TurbineIngressTimings is the per-slot decomposition carried only by a +// TurbineIngressTimings carries per-slot observations only on a // trusted in-memory Turbine block. Durations never serialize with Block. type TurbineIngressTimings struct { ShredCollection time.Duration CompletionQueueDelay time.Duration BlockDecode time.Duration + // Completion-only parse and outstanding-signature join/verification time. TransactionParse time.Duration TransactionSigverify time.Duration ReplayAdmission time.Duration + // Early durations sum completed prefetched component work, including an + // optimistic prefix later discarded, and overlap reception and each other. + // EarlyTransactionSigverify includes queueing through future completion; + // neither early duration is CPU time or an additive pipeline wall stage. + EarlyTransactionParse time.Duration + EarlyTransactionSigverify time.Duration + // Completion wait for already-claimed background parsing/submission. + // Recorded separately from BlockDecode's active completion work. + EarlyPreparationWait time.Duration + // Only retained transactions whose verification finished by ShredFullNanos. + EarlyVerifiedTransactions uint64 + // FullToReady is wall time from full shred assembly to replay-ready completion. + // It contains completion queueing, decode and outstanding verification waits. + FullToReady time.Duration } var transactionDerivedStateInitMu sync.Mutex diff --git a/pkg/block/block_test.go b/pkg/block/block_test.go index 2ffcd3029..31ac89e61 100644 --- a/pkg/block/block_test.go +++ b/pkg/block/block_test.go @@ -11,7 +11,12 @@ func TestTransactionSignaturesVerifiedMarkerIsNotSerialized(t *testing.T) { original.MarkTransactionSignaturesVerified() admissionStart := time.Now() original.MarkTurbineReplayAdmissionStart(admissionStart) - ingress := TurbineIngressTimings{ShredCollection: 12 * time.Millisecond, TransactionSigverify: 34 * time.Millisecond} + ingress := TurbineIngressTimings{ + ShredCollection: 12 * time.Millisecond, TransactionSigverify: 34 * time.Millisecond, + EarlyTransactionParse: 2 * time.Millisecond, EarlyTransactionSigverify: 56 * time.Millisecond, + EarlyPreparationWait: 3 * time.Millisecond, + EarlyVerifiedTransactions: 80, FullToReady: 35 * time.Millisecond, + } original.MarkTurbineIngressTimings(ingress) if got, ok := original.TurbineIngressTimings(); !ok || got != ingress { t.Fatalf("ingress timings = %+v, %t; want %+v, true", got, ok, ingress) diff --git a/pkg/block/verified_message_identity.go b/pkg/block/verified_message_identity.go new file mode 100644 index 000000000..9eff1d264 --- /dev/null +++ b/pkg/block/verified_message_identity.go @@ -0,0 +1,37 @@ +package block + +import ( + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" +) + +// CacheVerifiedTransactionMessageIdentities publishes identities from joined +// signature-verification requests. Every result must cover the exact ordered +// transaction slice. Failed/partial requests and obsolete prefetch generations +// cannot seed the cache. It does not replace block-wide duplicate/status checks. +func (b *Block) CacheVerifiedTransactionMessageIdentities(identities []txverify.VerifiedMessageIdentity) error { + if b == nil || len(identities) != len(b.Transactions) { + return fmt.Errorf("verified message identities do not cover block transactions") + } + prepared := &PreparedTransactionMessageIdentities{ + transactions: append([]*solana.Transaction(nil), b.Transactions...), + versions: make([]solana.MessageVersion, len(identities)), + identities: make([]txstatus.TransactionMessageIdentity, len(identities)), + } + for i, tx := range b.Transactions { + identity, ok := identities[i].ForTransaction(tx) + if !ok { + return fmt.Errorf("verified message identity does not match transaction %d", i) + } + prepared.versions[i] = tx.Message.GetVersion() + prepared.identities[i] = identity + } + state := b.transactionState() + state.mu.Lock() + defer state.mu.Unlock() + state.messageIdentities = prepared + return nil +} diff --git a/pkg/block/verified_message_identity_test.go b/pkg/block/verified_message_identity_test.go new file mode 100644 index 000000000..c82d5a4d2 --- /dev/null +++ b/pkg/block/verified_message_identity_test.go @@ -0,0 +1,42 @@ +package block + +import ( + "crypto/ed25519" + "encoding/json" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestVerifiedIdentityCacheOwnsStorageAndDoesNotSerializeTrust(t *testing.T) { + tx := identityTestTransaction(1) + key := ed25519.NewKeyFromSeed(make([]byte, 32)) + tx.Message.AccountKeys[0] = solana.PublicKeyFromBytes(key.Public().(ed25519.PublicKey)) + msg, err := tx.Message.MarshalBinary() + require.NoError(t, err) + tx.Signatures[0] = solana.SignatureFromBytes(ed25519.Sign(key, msg)) + blk := &Block{Transactions: []*solana.Transaction{tx}} + ids := make([]txverify.VerifiedMessageIdentity, 1) + errs := make([]error, 1) + var verifier txverify.BatchVerifier + verifier.VerifyWithMessageIdentities(blk.Transactions, errs, ids) + require.NoError(t, errs[0]) + require.NoError(t, blk.CacheVerifiedTransactionMessageIdentities(ids)) + cached := blk.transactionDerivedState.messageIdentities + require.NotNil(t, cached, "adoption must populate the cache before the first admission lookup") + clear(ids) + got, err := blk.PrepareTransactionMessageIdentities() + require.NoError(t, err) + require.Same(t, cached, got) + copyBlock := *blk + got, err = copyBlock.PrepareTransactionMessageIdentities() + require.NoError(t, err) + require.Same(t, cached, got) + wire, err := json.Marshal(blk) + require.NoError(t, err) + var decoded Block + require.NoError(t, json.Unmarshal(wire, &decoded)) + require.Nil(t, decoded.transactionDerivedState) +} diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index 26784fdf5..b93e07d48 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -3,6 +3,7 @@ package blockprod import ( "sync" + "github.com/Overclock-Validator/mithril/pkg/arena" "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/fees" @@ -44,6 +45,10 @@ type WorkingBank struct { // seenMessages is the bank-local AlreadyProcessed status set. The TPU's // signature LRU is only an ingress optimization and is not authoritative. seenMessages map[[32]byte]struct{} + // Execution and commit are serialized by mu. Borrowed accounts never escape + // either phase, so their storage can be reset for the next transaction. + borrowedAccounts *arena.Arena[sealevel.BorrowedAccount] + preparer *replay.TransactionPreparer } type BankConfig struct { @@ -71,7 +76,12 @@ func NewWorkingBank(cfg BankConfig) *WorkingBank { if sink == nil { sink = NopBatchSink{} } + var preparer *replay.TransactionPreparer + if cfg.SlotCtx != nil { + preparer = replay.NewTransactionPreparer(cfg.SlotCtx.Features) + } return &WorkingBank{ + preparer: preparer, slotCtx: cfg.SlotCtx, slot: cfg.Slot, leader: cfg.Leader, @@ -82,6 +92,7 @@ func NewWorkingBank(cfg BankConfig) *WorkingBank { accepting: true, ancestorStatuses: cfg.TransactionStatuses, seenMessages: make(map[[32]byte]struct{}), + borrowedAccounts: arena.New[sealevel.BorrowedAccount](64), } } @@ -175,11 +186,30 @@ func (b *WorkingBank) Forge(wire []byte) (ForgeResult, costmodel.ExceedReason) { // ForgeTransaction executes and commits a parsed transaction. func (b *WorkingBank) ForgeTransaction(tx *solana.Transaction, wireSize int) (ForgeResult, costmodel.ExceedReason) { + return b.forgeTransaction(tx, wireSize, nil) +} + +// ForgePreparedTransaction reuses static work from owned, immutable TPU bytes. +// A different bank feature snapshot falls back to the ordinary execution path. +func (b *WorkingBank) ForgePreparedTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction) (ForgeResult, costmodel.ExceedReason) { + if b.slotCtx == nil || !b.preparer.Matches(prepared, tx, b.slotCtx.Features) { + prepared = nil + } + return b.forgeTransaction(tx, wireSize, prepared) +} + +func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction) (ForgeResult, costmodel.ExceedReason) { if tx == nil { b.RebateSchedule(wireSize) return ForgeDroppedParse, costmodel.ExceedNone } - messageHash, err := replay.TransactionMessageHash(tx) + var messageHash [32]byte + var err error + if prepared != nil { + messageHash = prepared.MessageHash() + } else { + messageHash, err = replay.TransactionMessageHash(tx) + } if err != nil { b.RebateSchedule(wireSize) return ForgeDroppedParse, costmodel.ExceedNone @@ -201,7 +231,11 @@ func (b *WorkingBank) ForgeTransaction(tx *solana.Transaction, wireSize int) (Fo f := features.NewFeaturesDefault() feats = f } - cost, err = costmodel.EstimateTransactionCost(tx, feats) + if prepared != nil { + cost = prepared.Cost() + } else { + cost, err = costmodel.EstimateTransactionCost(tx, feats) + } if err != nil { b.RebateSchedule(wireSize) return ForgeDroppedParse, costmodel.ExceedNone @@ -238,15 +272,17 @@ func (b *WorkingBank) ForgeTransaction(tx *solana.Transaction, wireSize int) (Fo if reason := b.reserveEntryBytesLocked(wireSize); reason != costmodel.ExceedNone { return ForgeDroppedCost, reason } - if err := fees.PayerCanFund(b.slotCtx, tx); err != nil { + if err := b.preparer.PayerCanFund(b.slotCtx, tx, prepared); err != nil { return ForgeDroppedExecution, costmodel.ExceedNone } - output := replay.LoadAndExecuteTransaction(replay.LoadAndExecuteTransactionInput{ - SlotCtx: b.slotCtx, - Transaction: tx, - LeanResult: true, - }) + output := b.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ + SlotCtx: b.slotCtx, + Transaction: tx, + LeanResult: true, + SkipTimingMetrics: true, + Arena: b.borrowedAccounts, + }, prepared) if output.ProcessingResult.TransactionError != nil { feeInfo, err := replay.ApplyFeesOnlyTransaction(b.slotCtx, tx, output) if err != nil { diff --git a/pkg/blockprod/bank_test.go b/pkg/blockprod/bank_test.go index 53db0d5d4..98b9f7d13 100644 --- a/pkg/blockprod/bank_test.go +++ b/pkg/blockprod/bank_test.go @@ -797,6 +797,19 @@ func TestEntryBuilderFlush(t *testing.T) { assert.Greater(t, batchBytes, 0) } +func TestEntryBuilderDefaultTargetCoalescesTransactions(t *testing.T) { + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{0xcd}) + for seq := uint64(0); seq < 2; seq++ { + wire := txfixture.MustSignedTransferWire(seq) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + entries, _, flushed := builder.Append(*tx, len(wire)) + assert.False(t, flushed) + assert.Empty(t, entries) + } + assert.Equal(t, 2, builder.PendingCount()) +} + func TestControllerWorkingBank(t *testing.T) { controller := NewController() assert.Nil(t, controller.WorkingBank()) diff --git a/pkg/blockprod/entry.go b/pkg/blockprod/entry.go index 098e3170e..95e51d600 100644 --- a/pkg/blockprod/entry.go +++ b/pkg/blockprod/entry.go @@ -1,11 +1,10 @@ package blockprod import ( - "bytes" - "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/statsd" "github.com/Overclock-Validator/mithril/pkg/turbine" - bin "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" ) @@ -17,11 +16,12 @@ const entryBatchOverheadBytes = 8 + 8 + 32 + 8 type EntryBuilder struct { limits costmodel.Limits - pendingTxns []solana.Transaction - pendingWire int - flushedBytes int - reservedBytes int - entryHash solana.Hash + pendingTxns []solana.Transaction + pendingSerializedBytes int + pendingWire int + flushedBytes int + reservedBytes int + entryHash solana.Hash } func NewEntryBuilder(limits costmodel.Limits, entryHash solana.Hash) *EntryBuilder { @@ -102,32 +102,38 @@ func (b *EntryBuilder) dropReservation() { } // Append adds a forged transaction. The pending entry is held until the next -// transaction would overflow one FEC set. A short leftover is only emitted by -// Flush (slot end / Freeze). +// transaction would overflow the configured batch target. A short leftover is only emitted by +// Flush (slot end / Freeze). Appended transactions must remain immutable. func (b *EntryBuilder) Append(tx solana.Transaction, wireSize int) ([]turbine.Entry, int, bool) { + // Canonical component bytes may differ from a transport-size hint. Measure + // once per transaction and reuse the count at flush, preserving slot budgets. + wire, err := tx.MarshalBinary() + if err != nil { + _ = statsd.Count(statsd.BlockProductionEntrySerializationErrors, 1, nil) + mlog.Log.Errorf("entry builder: cannot serialize applied transaction: %v", err) + return nil, 0, false + } if wireSize <= 0 { - wire, err := tx.MarshalBinary() - if err != nil { - return nil, 0, false - } wireSize = len(wire) } b.consumeReserved(wireSize) - if b.wouldOverflowBatch(wireSize) { + if b.wouldOverflowBatch(len(wire)) { flushed, batchBytes := b.flushLocked() b.pendingTxns = append(b.pendingTxns[:0], tx) + b.pendingSerializedBytes = len(wire) b.pendingWire = wireSize return flushed, batchBytes, true } b.pendingTxns = append(b.pendingTxns, tx) + b.pendingSerializedBytes += len(wire) b.pendingWire += wireSize return nil, 0, false } func (b *EntryBuilder) projectedBytes(nextWire int) int { - return entryBatchOverheadBytes + b.pendingWire + nextWire + return entryBatchOverheadBytes + b.pendingSerializedBytes + nextWire } // Flush emits the current pending transactions as a single PoH entry. @@ -151,37 +157,10 @@ func (b *EntryBuilder) flushLocked() ([]turbine.Entry, int) { Txns: txns, }} b.entryHash = entryHash - batchBytes, err := marshalEntryBatchBytes(entries) - if err != nil { - return nil, 0 - } - b.flushedBytes += len(batchBytes) + batchBytes := entryBatchOverheadBytes + b.pendingSerializedBytes + b.flushedBytes += batchBytes b.pendingTxns = b.pendingTxns[:0] b.pendingWire = 0 - return entries, len(batchBytes) -} - -func marshalEntryBatchBytes(entries []turbine.Entry) ([]byte, error) { - var buf bytes.Buffer - enc := bin.NewEncoderWithEncoding(&buf, bin.EncodingBin) - if err := enc.WriteUint64(uint64(len(entries)), bin.LE); err != nil { - return nil, err - } - for _, entry := range entries { - if err := enc.WriteUint64(entry.NumHashes, bin.LE); err != nil { - return nil, err - } - if err := enc.WriteBytes(entry.Hash[:], false); err != nil { - return nil, err - } - if err := enc.WriteUint64(uint64(len(entry.Txns)), bin.LE); err != nil { - return nil, err - } - for i := range entry.Txns { - if err := entry.Txns[i].MarshalWithEncoder(enc); err != nil { - return nil, err - } - } - } - return buf.Bytes(), nil + b.pendingSerializedBytes = 0 + return entries, batchBytes } diff --git a/pkg/blockprod/entry_bench_test.go b/pkg/blockprod/entry_bench_test.go new file mode 100644 index 000000000..08f9bc5e7 --- /dev/null +++ b/pkg/blockprod/entry_bench_test.go @@ -0,0 +1,75 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" +) + +// Measure the producer's batch accounting both alone and through component +// serialization, shred generation, and broadcast-session bookkeeping. +// Transaction execution, peer routing, and UDP are excluded. +func BenchmarkEntryBuilder(b *testing.B) { + wires := txfixture.PrecomputeTransferPool(512) + txns := make([]solana.Transaction, len(wires)) + for i, wire := range wires { + tx, err := solana.TransactionFromBytes(wire) + if err != nil { + b.Fatal(err) + } + txns[i] = *tx + } + leader := txfixture.PayerPrivateKey() + for _, tc := range []struct { + name string + shred bool + broadcast bool + }{ + {name: "append"}, + {name: "append-and-shred", shred: true}, + {name: "append-and-broadcast", broadcast: true}, + } { + b.Run(tc.name, func(b *testing.B) { + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + shredder := turbine.Shredder{Slot: 100, ParentSlot: 99, Version: 1} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: 100, ParentSlot: 99, Version: 1, + Broadcaster: &benchmarkPacketBroadcaster{}, + }) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + index := i % len(txns) + entries, _, flushed := builder.Append(txns[index], len(wires[index])) + if flushed && tc.broadcast { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + } + if flushed && tc.shred { + component, err := turbine.NewEntryBatch(entries) + if err != nil { + b.Fatal(err) + } + if _, _, _, err := shredder.MakeMerkleShredsFromComponent( + leader, component, false, solana.Hash{}, 0, 0, + ); err != nil { + b.Fatal(err) + } + } + } + }) + } +} + +type benchmarkPacketBroadcaster struct { + packets int +} + +func (b *benchmarkPacketBroadcaster) Broadcast(packets [][]byte) error { + b.packets += len(packets) + return nil +} diff --git a/pkg/blockprod/entry_test.go b/pkg/blockprod/entry_test.go index 2a0f88fd9..a22ec5e45 100644 --- a/pkg/blockprod/entry_test.go +++ b/pkg/blockprod/entry_test.go @@ -5,6 +5,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/require" ) @@ -76,3 +77,102 @@ func mustTransferTx(t *testing.T, seq uint64) *solana.Transaction { require.NoError(t, err) return tx } + +func TestEntryBuilderBatchSizingMatchesComponentEncoding(t *testing.T) { + for _, tc := range []struct { + name string + count int + v0 bool + }{ + {name: "legacy", count: 2}, + {name: "mixed-version", count: 2, v0: true}, + {name: "more-than-128-transactions", count: 129, v0: true}, + } { + t.Run(tc.name, func(t *testing.T) { + txns := make([]solana.Transaction, tc.count) + for i := range txns { + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(uint64(i))) + require.NoError(t, err) + if tc.v0 && i%2 != 0 { + tx.Message.SetVersion(solana.MessageVersionV0) + } + txns[i] = *tx + } + component, err := turbine.NewEntryBatch([]turbine.Entry{{NumHashes: 1, Txns: txns}}) + require.NoError(t, err) + encoded, err := turbine.MarshalBlockComponent(component) + require.NoError(t, err) + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = uint64(len(encoded)) + builder := NewEntryBuilder(limits, solana.Hash{1}) + + totalFlushed := 0 + checkBatch := func(entries []turbine.Entry, batchBytes int, want []solana.Transaction) { + t.Helper() + require.Len(t, entries, 1) + require.Equal(t, want, entries[0].Txns) + component, err := turbine.NewEntryBatch(entries) + require.NoError(t, err) + encoded, err := turbine.MarshalBlockComponent(component) + require.NoError(t, err) + require.Equal(t, len(encoded), batchBytes) + totalFlushed += batchBytes + require.Equal(t, totalFlushed, builder.FlushedBytes()) + } + + // Repeat after both an automatic and explicit flush to catch stale + // size accounting. The limit fits the complete batch exactly. + for round := 0; round < 2; round++ { + pendingWire := 0 + for i := range txns { + wire, err := txns[i].MarshalBinary() + require.NoError(t, err) + wireHint := len(wire) + switch i % 3 { + case 0: + wireHint += 100 // Transport bytes must not affect batch sizing. + case 1: + wireHint = 0 // An absent hint must use the serialized length. + } + if wireHint > 0 { + pendingWire += wireHint + } else { + pendingWire += len(wire) + } + entries, _, flushed := builder.Append(txns[i], wireHint) + require.False(t, flushed, "transaction %d", i) + require.Empty(t, entries) + } + require.Equal(t, tc.count, builder.PendingCount()) + require.Equal(t, pendingWire, builder.PendingWireBytes()) + require.Equal(t, len(encoded), builder.projectedBytes(0)) + entries, batchBytes, flushed := builder.Append(txns[0], 0) + require.True(t, flushed) + checkBatch(entries, batchBytes, txns) + require.Equal(t, int(limits.MaxBatchBytes), batchBytes) + require.Equal(t, 1, builder.PendingCount()) + entries, batchBytes = builder.Flush() + checkBatch(entries, batchBytes, txns[:1]) + require.Zero(t, builder.PendingCount()) + require.Zero(t, builder.PendingWireBytes()) + } + }) + } +} + +func TestEntryBuilderAllowsSingleTransactionOverBatchTarget(t *testing.T) { + wire := txfixture.MustSignedTransferWire(0) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = 1 + builder := NewEntryBuilder(limits, solana.Hash{}) + entries, _, flushed := builder.Append(*tx, len(wire)) + require.False(t, flushed) + require.Empty(t, entries) + entries, _, flushed = builder.Append(*tx, len(wire)) + require.True(t, flushed) + require.Len(t, entries, 1) + require.Len(t, entries[0].Txns, 1) + require.Equal(t, 1, builder.PendingCount()) +} diff --git a/pkg/blockprod/leader.go b/pkg/blockprod/leader.go index e304ffb8d..66bce23ee 100644 --- a/pkg/blockprod/leader.go +++ b/pkg/blockprod/leader.go @@ -103,11 +103,12 @@ type LeaderLoop struct { currentSlot func() uint64 leaderForSlot func(uint64) (solana.PublicKey, bool) - pollInterval time.Duration - slotDuration time.Duration - now func() time.Time - tickStartedAt time.Time - tickDeliveryLag time.Duration + pollInterval time.Duration + slotDuration time.Duration + completionReserve time.Duration + now func() time.Time + tickStartedAt time.Time + tickDeliveryLag time.Duration mu sync.Mutex activeSlot uint64 @@ -154,7 +155,10 @@ type LeaderLoopConfig struct { RewardCerts RewardCertBuilder PollInterval time.Duration SlotDuration time.Duration - Now func() time.Time + // CompletionReserve is time retained for finalization and broadcast. Zero + // uses the conservative default; tune only from measured completion times. + CompletionReserve time.Duration + Now func() time.Time } func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { @@ -170,6 +174,9 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { if cfg.Now == nil { cfg.Now = time.Now } + if cfg.CompletionReserve <= 0 { + cfg.CompletionReserve = leaderBlockCompletionReserve + } return &LeaderLoop{ controller: cfg.Controller, identity: cfg.Identity, @@ -187,6 +194,7 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { rewardCerts: cfg.RewardCerts, pollInterval: cfg.PollInterval, slotDuration: cfg.SlotDuration, + completionReserve: cfg.CompletionReserve, now: cfg.Now, finishedLeaderSlots: make(map[uint64]struct{}), pendingFailures: make(map[uint64]leaderSlotFailure), @@ -329,9 +337,11 @@ func (l *LeaderLoop) tickScheduled(scheduledAt time.Time) { delete(l.pendingFailures, targetSlot) openedAt := l.now() l.recordTickTimingLocked(openedAt, "opened") - mlog.Log.InfofPrecise("ALPENGLOW block production: opened local leader slot=%d parent_slot=%d replay_frontier=%d live_slot=%d start_slot_ms=%d%s", + limits := l.activeBank.CostTracker().Limits() + mlog.Log.InfofPrecise("ALPENGLOW block production: opened local leader slot=%d parent_slot=%d replay_frontier=%d live_slot=%d start_slot_ms=%d block_cost_limit=%d account_cost_limit=%d entry_bytes_limit=%d%s", targetSlot, l.parentCtx.ParentSlot, global.ReplayFrontier(), wallSlot, - startDuration.Milliseconds(), l.productionStartTimingDetailLocked(targetSlot, openedAt)) + startDuration.Milliseconds(), limits.BlockCost, limits.WritableAccountCost, limits.MaxEntryBytes, + l.productionStartTimingDetailLocked(targetSlot, openedAt)) return } @@ -586,7 +596,10 @@ func (l *LeaderLoop) productionWindowDeadlineLocked(slot uint64) time.Time { if deadline.IsZero() { return time.Time{} } - reserve := leaderBlockCompletionReserve + reserve := l.completionReserve + if reserve <= 0 { + reserve = leaderBlockCompletionReserve + } if reserve >= l.slotDuration { reserve = l.slotDuration / 4 } @@ -1205,12 +1218,16 @@ func (l *LeaderLoop) startSlotLocked(slot uint64) error { if startEntryHash == (solana.Hash{}) { startEntryHash = parentCtx.ParentBankhash } + limits, err := costmodel.LimitsForSlot(slotCtx.Features, epochSchedule, slot) + if err != nil { + return fmt.Errorf("leader slot limits: %w", err) + } sink := NewShredSink(session) bank := NewWorkingBank(BankConfig{ SlotCtx: slotCtx, Slot: slot, Leader: l.identity.PublicKey(), - Limits: costmodel.LimitsForFeatures(slotCtx.Features), + Limits: limits, EntryHash: startEntryHash, Sink: sink, TransactionStatuses: parentCtx.TransactionStatuses, diff --git a/pkg/blockprod/leader_completion_reserve_test.go b/pkg/blockprod/leader_completion_reserve_test.go new file mode 100644 index 000000000..062bf84d4 --- /dev/null +++ b/pkg/blockprod/leader_completion_reserve_test.go @@ -0,0 +1,32 @@ +package blockprod + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestConfiguredCompletionReserveKeepsProtocolDeadlines(t *testing.T) { + ready := time.Unix(1_700_000_000, 0) + now := ready + loop := NewLeaderLoop(LeaderLoopConfig{ + SlotDuration: AlpenglowSlotDuration, CompletionReserve: 60 * time.Millisecond, + Now: func() time.Time { return now }, + }) + loop.productionWindow = leaderProductionWindow{ + active: true, startSlot: 212, endSlot: 215, nextSlot: 212, readyAt: ready, + } + for offset := uint64(0); offset < 4; offset++ { + slot := 212 + offset + deadline := ready.Add(time.Duration(offset+1) * 200 * time.Millisecond) + require.Equal(t, deadline, loop.productionWindowProtocolDeadlineLocked(slot)) + require.Equal(t, deadline.Add(-60*time.Millisecond), loop.productionWindowDeadlineLocked(slot)) + // The override admits starts during the additional 15ms, but still + // rejects at its own cutoff instead of extending the protocol deadline. + now = deadline.Add(-70 * time.Millisecond) + require.NoError(t, loop.productionStartCutoffErrorLocked(slot)) + now = deadline.Add(-60 * time.Millisecond) + require.ErrorIs(t, loop.productionStartCutoffErrorLocked(slot), errProductionStartCutoffElapsed) + } +} diff --git a/pkg/blockprod/leader_processing_test.go b/pkg/blockprod/leader_processing_test.go new file mode 100644 index 000000000..71303a2cf --- /dev/null +++ b/pkg/blockprod/leader_processing_test.go @@ -0,0 +1,81 @@ +package blockprod + +import ( + "sync/atomic" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/arena" + "github.com/Overclock-Validator/mithril/pkg/metrics" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestLeaderExecutionOptionsPreserveResults(t *testing.T) { + for _, testCase := range []string{"success", "instruction_failure", "payer_failure"} { + t.Run(testCase, func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(42)) + require.NoError(t, err) + if testCase == "instruction_failure" { + tx.Message.Instructions[0].Data[0] = 0xff + } + if testCase == "payer_failure" { + setPayerLamports(t, env, 1) + } + input := replay.LoadAndExecuteTransactionInput{SlotCtx: env.SlotCtx, Transaction: tx, LeanResult: true} + reference := replay.LoadAndExecuteTransaction(input) + input.SkipTimingMetrics = true + // A one-object arena also exercises the heap fallback. Repeated calls + // reset it, while the previous result's account state remains valid. + input.Arena = arena.New[sealevel.BorrowedAccount](1) + before := atomic.LoadUint64(&metrics.GlobalBlockReplay.InstructionsAndAccountMetasFromTx.Count) + beforeDispatch := atomic.LoadUint64(&metrics.GlobalBlockReplay.GetNextIxCtx.Count) + for i := 0; i < 3; i++ { + got := replay.LoadAndExecuteTransaction(input) + require.Equal(t, reference.ProcessingResult, got.ProcessingResult) + require.Equal(t, reference.FeeInfo, got.FeeInfo) + require.Equal(t, reference.LoadedAccountsDataSize, got.LoadedAccountsDataSize) + if reference.ExecCtx != nil { + require.NotNil(t, got.ExecCtx) + require.Equal(t, reference.ExecCtx.ComputeMeter.Used(), got.ExecCtx.ComputeMeter.Used()) + require.Equal(t, reference.ExecCtx.TransactionContext.Accounts.Accounts, got.ExecCtx.TransactionContext.Accounts.Accounts) + } + } + require.Equal(t, before, atomic.LoadUint64(&metrics.GlobalBlockReplay.InstructionsAndAccountMetasFromTx.Count)) + require.Equal(t, beforeDispatch, atomic.LoadUint64(&metrics.GlobalBlockReplay.GetNextIxCtx.Count)) + }) + } +} + +func TestWorkingBankReusesBorrowedAccountsWithoutChangingMessages(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore, err := env.SlotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + payerBalance := payerBefore.Lamports + destBefore, err := env.SlotCtx.GetAccount(txfixture.DestPubkey()) + require.NoError(t, err) + destBalance := destBefore.Lamports + const count = 130 + for i := 0; i < count; i++ { + wire := txfixture.MustSignedTransferWire(uint64(i)) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + result, _ := env.Bank.ForgeTransaction(tx, len(wire)) + require.Equal(t, ForgeAccepted, result) + after, err := tx.MarshalBinary() + require.NoError(t, err) + require.Equal(t, wire, after) + } + payerAfter, err := env.SlotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + destAfter, err := env.SlotCtx.GetAccount(txfixture.DestPubkey()) + require.NoError(t, err) + const transferred = count * (count + 1) / 2 + require.Equal(t, payerBalance-transferred-count*5000, payerAfter.Lamports) + require.Equal(t, destBalance+transferred, destAfter.Lamports) +} diff --git a/pkg/blockprod/leader_throughput_bench_test.go b/pkg/blockprod/leader_throughput_bench_test.go new file mode 100644 index 000000000..006633c93 --- /dev/null +++ b/pkg/blockprod/leader_throughput_bench_test.go @@ -0,0 +1,117 @@ +package blockprod + +import ( + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + computebudget "github.com/gagliardetto/solana-go/programs/compute-budget" +) + +// BenchmarkWorkingBankHotAccounts measures admission, execution, publication, +// and entry batching. Signing and bank setup are outside the measured region. +// All wires are unique within a bank, so this cannot benchmark dedup rejection. +func BenchmarkWorkingBankHotAccounts(b *testing.B) { benchmarkWorkingBankHotAccounts(b, "wire") } +func BenchmarkWorkingBankDecodedHotAccounts(b *testing.B) { + benchmarkWorkingBankHotAccounts(b, "decoded") +} +func BenchmarkWorkingBankPreparedHotAccounts(b *testing.B) { + benchmarkWorkingBankHotAccounts(b, "prepared") +} + +func benchmarkWorkingBankHotAccounts(b *testing.B, mode string) { + const perBank = 10000 + for _, workload := range []string{"compute_budget", "transfer"} { + b.Run(workload, func(b *testing.B) { + wires := make([][]byte, perBank) + for i := range wires { + if workload == "transfer" { + wires[i] = txfixture.MustSignedTransferWire(uint64(i)) + continue + } + tx, err := solana.NewTransaction([]solana.Instruction{ + computebudget.NewSetComputeUnitLimitInstruction(uint32(1000 + i)).Build(), + }, txfixture.TestBlockhash(), solana.TransactionPayer(txfixture.PayerPubkey())) + if err != nil { + b.Fatal(err) + } + key := txfixture.PayerPrivateKey() + if _, err = tx.Sign(func(solana.PublicKey) *solana.PrivateKey { return &key }); err != nil { + b.Fatal(err) + } + wires[i], err = tx.MarshalBinary() + if err != nil { + b.Fatal(err) + } + } + decoded := make([]*solana.Transaction, perBank) + for i, wire := range wires { + var err error + decoded[i], err = solana.TransactionFromBytes(wire) + if err != nil { + b.Fatal(err) + } + } + var prepared []*replay.PreparedTransaction + var env *TestEnv + defer func() { + if env != nil { + env.Close() + } + }() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i%perBank == 0 { + b.StopTimer() + if env != nil { + env.Close() + } + env = NewTestEnv(TestEnvConfig{}) + env.SlotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) + env.SlotCtx.Features.EnableFeature(features.RaiseBlockLimitsTo100m, 0) + if err := env.SlotCtx.Accounts.SetAccount(&addresses.ComputeBudgetProgramAddr, &accounts.Account{ + Key: addresses.ComputeBudgetProgramAddr, Lamports: 1, + Owner: addresses.NativeLoaderAddr, Executable: true, RentEpoch: math.MaxUint64, + }); err != nil { + b.Fatal(err) + } + + if mode == "prepared" { + // Bind the immutable snapshot after all fixture feature setup is complete. + env.Bank.preparer = replay.NewTransactionPreparer(env.SlotCtx.Features) + if prepared == nil { + prepared = make([]*replay.PreparedTransaction, perBank) + for j, tx := range decoded { + prepared[j] = env.Bank.preparer.Prepare(tx) + if prepared[j] == nil { + b.Fatal("preparation failed") + } + } + } + } + b.StartTimer() + } + var result ForgeResult + var reason costmodel.ExceedReason + switch mode { + case "prepared": + result, reason = env.Bank.ForgePreparedTransaction(decoded[i%perBank], len(wires[i%perBank]), prepared[i%perBank]) + case "decoded": + result, reason = env.Bank.ForgeTransaction(decoded[i%perBank], len(wires[i%perBank])) + default: + result, reason = env.Bank.Forge(wires[i%perBank]) + } + if result != ForgeAccepted { + b.Fatalf("transaction %d: %v / %v", i, result, reason) + } + } + }) + } +} diff --git a/pkg/blockprod/prepared_transaction_test.go b/pkg/blockprod/prepared_transaction_test.go new file mode 100644 index 000000000..9a6153d64 --- /dev/null +++ b/pkg/blockprod/prepared_transaction_test.go @@ -0,0 +1,142 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + computebudget "github.com/gagliardetto/solana-go/programs/compute-budget" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/require" +) + +func TestPreparedBankPreservesOutcomes(t *testing.T) { + for _, kind := range []string{"success", "instruction_failure", "fees_only", "payer_changed", "expired", "duplicate", "foreign_message"} { + t.Run(kind, func(t *testing.T) { + reference := NewTestEnv(TestEnvConfig{}) + defer reference.Close() + candidate := NewTestEnv(TestEnvConfig{}) + defer candidate.Close() + tx := mustSignedTransfer(t, 7) + if kind == "instruction_failure" { + tx.Message.Instructions[0].Data[0] = 0xff + } + if kind == "fees_only" { + tx = mustSignBankTestTransaction(t, + computebudget.NewSetLoadedAccountsDataSizeLimitInstruction(1).Build(), + system.NewTransferInstruction(1, txfixture.PayerPubkey(), txfixture.DestPubkey()).Build()) + } + if kind == "expired" { + tx.Message.RecentBlockhash = solana.Hash{99} + } + prepared := replay.NewTransactionPreparer(candidate.SlotCtx.Features.Clone()).Prepare(tx) + require.NotNil(t, prepared) + estimate, err := costmodel.EstimateTransactionCost(tx, candidate.SlotCtx.Features) + require.NoError(t, err) + require.Equal(t, estimate, prepared.Cost()) + if kind == "payer_changed" { + // Preparation succeeded while the payer was funded. Admission must + // still reject after another transaction spends that balance. + setPayerLamports(t, reference, 1) + setPayerLamports(t, candidate, 1) + } + if kind == "foreign_message" { + tx = mustSignedTransfer(t, 19) + } + wire, err := tx.MarshalBinary() + require.NoError(t, err) + for i := 0; i < 2; i++ { + a, ar := reference.Bank.ForgeTransaction(tx, len(wire)) + b, br := candidate.Bank.ForgePreparedTransaction(tx, len(wire), prepared) + require.Equal(t, a, b) + require.Equal(t, ar, br) + require.Equal(t, reference.Bank.CostTracker().BlockCost(), candidate.Bank.CostTracker().BlockCost()) + require.Equal(t, reference.Bank.TxFeeAccumulator(), candidate.Bank.TxFeeAccumulator()) + for _, key := range []solana.PublicKey{txfixture.PayerPubkey(), txfixture.DestPubkey()} { + ra, err := reference.SlotCtx.GetAccount(key) + require.NoError(t, err) + ca, err := candidate.SlotCtx.GetAccount(key) + require.NoError(t, err) + require.Equal(t, ra, ca) + } + if kind != "duplicate" { + break + } + } + after, err := tx.MarshalBinary() + require.NoError(t, err) + require.Equal(t, wire, after) + }) + } +} + +func TestPreparedBankRejectsStaleFeatures(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + future := env.SlotCtx.Features.Clone() + future.EnableFeature(features.EnableTxV1, 0) + tx := mustSignedTransfer(t, 1) + _, err := tx.Message.SetVersion(solana.MessageVersionV1) + require.NoError(t, err) + prepared := replay.NewTransactionPreparer(future).Prepare(tx) + require.NotNil(t, prepared) + require.False(t, env.Bank.preparer.Matches(prepared, tx, env.SlotCtx.Features)) + wire, err := tx.MarshalBinary() + require.NoError(t, err) + result, _ := env.Bank.ForgePreparedTransaction(tx, len(wire), prepared) + require.Equal(t, ForgeDroppedExecution, result) + require.Empty(t, env.Bank.ForgedTransactions()) + require.Zero(t, env.Bank.TxFeeAccumulator().TotalFees) +} + +func TestPreparedLeaderStillRejectsFeePayerNoOp(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + // Establish the complete feature snapshot before constructing the bank. + env.SlotCtx.Features.EnableFeature(features.RelaxFeePayerConstraint, 0) + bank := NewWorkingBank(BankConfig{SlotCtx: env.SlotCtx, Slot: env.SlotCtx.Slot, + TransactionStatuses: replay.NewTransactionStatusCache().View()}) + tx := mustSignedTransfer(t, 1) + prepared := bank.preparer.Prepare(tx) + require.NotNil(t, prepared) + setPayerLamports(t, env, 1) + preview := bank.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ + SlotCtx: env.SlotCtx, Transaction: tx, LeanResult: true}, prepared) + require.True(t, preview.ProcessedAsNoOp) + wire, err := tx.MarshalBinary() + require.NoError(t, err) + result, _ := bank.ForgePreparedTransaction(tx, len(wire), prepared) + require.Equal(t, ForgeDroppedExecution, result) + require.Empty(t, bank.ForgedTransactions()) + require.Zero(t, bank.TxFeeAccumulator().TotalFees) +} + +func TestPreparedExecutionRechecksStateWithoutMutatingPreparation(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + tx := mustSignedTransfer(t, 1) + p := replay.NewTransactionPreparer(env.SlotCtx.Features) + prepared := p.Prepare(tx) + require.NotNil(t, prepared) + input := replay.LoadAndExecuteTransactionInput{SlotCtx: env.SlotCtx, Transaction: tx, LeanResult: true} + for _, balance := range []uint64{10_000_000, 1, 20_000_000} { + setPayerLamports(t, env, balance) + got := p.LoadAndExecute(input, prepared) + want := replay.LoadAndExecuteTransaction(input) + require.Equal(t, want.ProcessingResult, got.ProcessingResult) + require.Equal(t, want.FeeInfo, got.FeeInfo) + require.Equal(t, want.LoadedAccountsDataSize, got.LoadedAccountsDataSize) + if want.ExecCtx != nil { + require.Equal(t, want.ExecCtx.TransactionContext.Accounts.Accounts, got.ExecCtx.TransactionContext.Accounts.Accounts) + require.Equal(t, want.ExecCtx.ComputeMeter.Used(), got.ExecCtx.ComputeMeter.Used()) + } + } + // Rent-boundary checks still use the bank's current payer state. + rent := sealevel.NewDefaultRentSysvar() + setPayerLamports(t, env, rent.MinimumBalance(0)+4999) + require.Error(t, p.PayerCanFund(env.SlotCtx, tx, prepared)) +} diff --git a/pkg/blockprod/producer_block_bench_test.go b/pkg/blockprod/producer_block_bench_test.go new file mode 100644 index 000000000..c93c6849d --- /dev/null +++ b/pkg/blockprod/producer_block_bench_test.go @@ -0,0 +1,66 @@ +package blockprod + +import ( + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "testing" +) + +func BenchmarkProducerBlock50k(b *testing.B) { + const transactions = 50000 + wires := txfixture.PrecomputeTransferPool(512) + txns := make([]solana.Transaction, len(wires)) + for i, wire := range wires { + tx, err := solana.TransactionFromBytes(wire) + if err != nil { + b.Fatal(err) + } + txns[i] = *tx + } + leader := txfixture.PayerPrivateKey() + var lastBatches, lastPackets int + b.ReportAllocs() + b.ResetTimer() + for round := 0; round < b.N; round++ { + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + sink := &benchmarkPacketBroadcaster{} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: 100, ParentSlot: 99, Version: 1, Broadcaster: sink, + }) + if err := session.BroadcastHeader(solana.Hash{}); err != nil { + b.Fatal(err) + } + batches := 0 + for i := 0; i < transactions; i++ { + idx := i % len(txns) + entries, _, flushed := builder.Append(txns[idx], len(wires[idx])) + if flushed { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + batches++ + } + } + if entries, _ := builder.Flush(); len(entries) > 0 { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + batches++ + } + if err := session.BroadcastFooter(solana.Hash{1}, 0, nil, nil); err != nil { + b.Fatal(err) + } + if err := session.BroadcastEndingTickLast(builder.CurrentEntryHash()); err != nil { + b.Fatal(err) + } + _ = session.BlockID(99, solana.Hash{}) + lastBatches, lastPackets = batches, sink.packets + } + b.StopTimer() + b.ReportMetric(transactions, "transactions/op") + b.ReportMetric(float64(len(wires[0])), "wire-B/tx") + b.ReportMetric(float64(lastBatches), "entry-batches/op") + b.ReportMetric(float64(lastPackets), "packets/op") +} diff --git a/pkg/blockprod/producer_branch_bench_test.go b/pkg/blockprod/producer_branch_bench_test.go new file mode 100644 index 000000000..93b9d92e6 --- /dev/null +++ b/pkg/blockprod/producer_branch_bench_test.go @@ -0,0 +1,253 @@ +package blockprod + +import ( + "bytes" + "fmt" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + txwire "github.com/Overclock-Validator/mithril/pkg/tpu/wire" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" +) + +// These complete producer CPU workloads compare branch heads using identical +// pre-signed, parsed fixtures and a packet-counting sink. No execution, admission, +// worker queue, routing, UDP, or receiver work is timed. +const branchComparisonOneFECBytes = 32 * 963 +const branchComparisonSlotEntryCap = 20*1024*1024 - 48 + +func branchMaxSizeTransferPool(tb testing.TB) ([][]byte, []solana.Transaction) { + tb.Helper() + payer, destination := txfixture.PayerPubkey(), txfixture.DestPubkey() + privateKey := txfixture.PayerPrivateKey() + wires := make([][]byte, 512) + txns := make([]solana.Transaction, len(wires)) + for i := range wires { + memo := bytes.Repeat([]byte("m"), 981) + copy(memo, fmt.Sprintf("mithril-max-wire-%04d:", i)) + tx, err := solana.NewTransaction([]solana.Instruction{ + system.NewTransferInstruction(uint64(i+1), payer, destination).Build(), + solana.NewInstruction(solana.MemoProgramID, nil, memo), + }, txfixture.TestBlockhash(), solana.TransactionPayer(payer)) + if err != nil { + tb.Fatal(err) + } + _, err = tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key == payer { + return &privateKey + } + return nil + }) + if err != nil { + tb.Fatal(err) + } + raw, err := tx.MarshalBinary() + if err != nil { + tb.Fatal(err) + } + if len(raw) != costmodel.PacketDataSize { + tb.Fatalf("wire size %d, expected %d", len(raw), costmodel.PacketDataSize) + } + if _, err := txwire.Sanitize(raw); err != nil { + tb.Fatal(err) + } + decoded, err := solana.TransactionFromBytes(raw) + if err != nil { + tb.Fatal(err) + } + if err := decoded.VerifySignatures(); err != nil { + tb.Fatal(err) + } + encoded, err := decoded.MarshalBinary() + if err != nil || !bytes.Equal(encoded, raw) { + tb.Fatal("canonical transaction round trip changed bytes") + } + wires[i], txns[i] = raw, *decoded + } + return wires, txns +} + +func branchComparisonFixtures(tb testing.TB, maximum bool) ([][]byte, []solana.Transaction) { + tb.Helper() + if maximum { + return branchMaxSizeTransferPool(tb) + } + wires := txfixture.PrecomputeTransferPool(512) + txns := make([]solana.Transaction, len(wires)) + for i, raw := range wires { + if len(raw) != 215 { + tb.Fatalf("transfer size %d, want 215", len(raw)) + } + if _, err := txwire.Sanitize(raw); err != nil { + tb.Fatal(err) + } + tx, err := solana.TransactionFromBytes(raw) + if err != nil { + tb.Fatal(err) + } + if err := tx.VerifySignatures(); err != nil { + tb.Fatal(err) + } + encoded, err := tx.MarshalBinary() + if err != nil || !bytes.Equal(encoded, raw) { + tb.Fatal("transfer round trip changed bytes") + } + txns[i] = *tx + } + return wires, txns +} + +type branchComparisonSink struct{ packets int } + +func (s *branchComparisonSink) Broadcast(packets [][]byte) error { + s.packets += len(packets) + return nil +} + +type branchComparisonStats struct { + batches, packets, transactions, maxDataShreds, maxEntryBytes int + blockID solana.Hash +} + +func branchComparisonRun(tb testing.TB, wires [][]byte, txns []solana.Transaction, slots []int, limits costmodel.Limits) branchComparisonStats { + tb.Helper() + var stats branchComparisonStats + var parentID, parentRoot solana.Hash + leader := txfixture.PayerPrivateKey() + for number, count := range slots { + slot := uint64(100 + number) + builder := NewEntryBuilder(limits, solana.Hash{}) + sink := &branchComparisonSink{} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: slot, ParentSlot: slot - 1, Version: 1, Broadcaster: sink, + ParentBlockID: parentID, ParentChainedMerkleRoot: parentRoot, + }) + if err := session.BroadcastHeader(parentID); err != nil { + tb.Fatal(err) + } + entryBytes := 0 + for i := 0; i < count; i++ { + index := stats.transactions % len(txns) + entries, size, flushed := builder.Append(txns[index], len(wires[index])) + stats.transactions++ + if flushed { + if err := session.BroadcastEntryBatch(entries); err != nil { + tb.Fatal(err) + } + stats.batches++ + entryBytes += size + } + } + if entries, size := builder.Flush(); len(entries) != 0 { + if err := session.BroadcastEntryBatch(entries); err != nil { + tb.Fatal(err) + } + stats.batches++ + entryBytes += size + } + if err := session.BroadcastFooter(solana.Hash{1}, 0, nil, nil); err != nil { + tb.Fatal(err) + } + if err := session.BroadcastEndingTickLast(builder.CurrentEntryHash()); err != nil { + tb.Fatal(err) + } + parentID = session.BlockID(slot-1, parentID) + parentRoot = session.ChainedMerkleRoot() + dataShreds := sink.packets / 2 + if dataShreds > costmodel.DefaultMaxDataShredsPerSlot { + tb.Fatal("slot exceeds data-shred budget") + } + if entryBytes > branchComparisonSlotEntryCap { + tb.Fatal("slot exceeds entry-byte budget") + } + if dataShreds > stats.maxDataShreds { + stats.maxDataShreds = dataShreds + } + if entryBytes > stats.maxEntryBytes { + stats.maxEntryBytes = entryBytes + } + stats.packets += sink.packets + } + stats.blockID = parentID + return stats +} + +func branchComparisonCheck(tb testing.TB, got branchComparisonStats, wireSize int, slots []int, batchLimit uint64) { + tb.Helper() + perBatch := (int(batchLimit) - 56) / wireSize + var batches, packets, transactions int + for _, count := range slots { + full, tail := count/perBatch, count%perBatch + batches += full + fecs := full * ((56 + perBatch*wireSize + branchComparisonOneFECBytes - 1) / branchComparisonOneFECBytes) + if tail != 0 { + batches++ + fecs += (56 + tail*wireSize + branchComparisonOneFECBytes - 1) / branchComparisonOneFECBytes + } + packets += (fecs + 3) * 64 // Header, footer, and signed ending tick each use one FEC set. + transactions += count + } + if got.batches != batches || got.packets != packets || got.transactions != transactions { + tb.Fatalf("workload counts %+v, want batches=%d packets=%d transactions=%d", got, batches, packets, transactions) + } + if got.blockID == (solana.Hash{}) { + tb.Fatal("empty block commitment") + } +} + +func TestProducerBranchComparisonFixtures(t *testing.T) { + for _, maximum := range []bool{false, true} { + wires, txns := branchComparisonFixtures(t, maximum) + limits := costmodel.DefaultLimits() + slots := []int{101, 100} + got := branchComparisonRun(t, wires, txns, slots, limits) + branchComparisonCheck(t, got, len(wires[0]), slots, limits.MaxBatchBytes) + t.Logf("wire=%d batch-target=%d batches=%d packets=%d", len(wires[0]), limits.MaxBatchBytes, got.batches, got.packets) + } +} + +func BenchmarkProducerBranchHeads(b *testing.B) { + for _, workload := range []struct { + name string + maximum bool + slots []int + }{ + {name: "small-215B", slots: []int{50000}}, + {name: "maximum-1232B", maximum: true, slots: []int{16667, 16667, 16666}}, + } { + wires, txns := branchComparisonFixtures(b, workload.maximum) + for _, target := range []struct { + name string + bytes uint64 + }{ + {name: "default"}, + {name: "one-fec", bytes: branchComparisonOneFECBytes}, + } { + b.Run(workload.name+"/"+target.name, func(b *testing.B) { + limits := costmodel.DefaultLimits() + if target.bytes != 0 { + limits.MaxBatchBytes = target.bytes + } + var last branchComparisonStats + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + last = branchComparisonRun(b, wires, txns, workload.slots, limits) + } + b.StopTimer() + branchComparisonCheck(b, last, len(wires[0]), workload.slots, limits.MaxBatchBytes) + b.ReportMetric(float64(last.transactions), "transactions/op") + b.ReportMetric(float64(len(workload.slots)), "slots/op") + b.ReportMetric(float64(len(wires[0])), "wire-B/tx") + b.ReportMetric(float64(limits.MaxBatchBytes), "batch-target-B") + b.ReportMetric(float64(last.batches), "entry-batches/op") + b.ReportMetric(float64(last.packets), "packets/op") + b.ReportMetric(float64(last.maxDataShreds), "max-data-shreds/slot") + b.ReportMetric(float64(last.maxEntryBytes), "max-entry-B/slot") + }) + } + } +} diff --git a/pkg/blockprod/producer_maxsize_bench_test.go b/pkg/blockprod/producer_maxsize_bench_test.go new file mode 100644 index 000000000..e843baf81 --- /dev/null +++ b/pkg/blockprod/producer_maxsize_bench_test.go @@ -0,0 +1,153 @@ +package blockprod + +import ( + "bytes" + "fmt" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + txwire "github.com/Overclock-Validator/mithril/pkg/tpu/wire" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" +) + +// Generate exactly 1,232-byte, single-signature legacy transactions. Padding is +// a real UTF-8 memo instruction, rather than trailing bytes after a transaction. +func maxSizeTransferPool(tb testing.TB) ([][]byte, []solana.Transaction) { + tb.Helper() + payer, destination := txfixture.PayerPubkey(), txfixture.DestPubkey() + privateKey := txfixture.PayerPrivateKey() + wires := make([][]byte, 512) + txns := make([]solana.Transaction, len(wires)) + for i := range wires { + memo := bytes.Repeat([]byte("m"), 981) + copy(memo, fmt.Sprintf("mithril-max-wire-%04d:", i)) + tx, err := solana.NewTransaction([]solana.Instruction{ + system.NewTransferInstruction(uint64(i+1), payer, destination).Build(), + solana.NewInstruction(solana.MemoProgramID, nil, memo), + }, txfixture.TestBlockhash(), solana.TransactionPayer(payer)) + if err != nil { + tb.Fatal(err) + } + _, err = tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key == payer { + return &privateKey + } + return nil + }) + if err != nil { + tb.Fatal(err) + } + raw, err := tx.MarshalBinary() + if err != nil { + tb.Fatal(err) + } + if len(raw) != costmodel.PacketDataSize { + tb.Fatalf("wire size %d, expected %d", len(raw), costmodel.PacketDataSize) + } + if _, err := txwire.Sanitize(raw); err != nil { + tb.Fatal(err) + } + decoded, err := solana.TransactionFromBytes(raw) + if err != nil { + tb.Fatal(err) + } + if err := decoded.VerifySignatures(); err != nil { + tb.Fatal(err) + } + encoded, err := decoded.MarshalBinary() + if err != nil || !bytes.Equal(encoded, raw) { + tb.Fatal("canonical transaction round trip changed bytes") + } + wires[i], txns[i] = raw, *decoded + } + return wires, txns +} + +func TestMaxSizeProducerFixture(t *testing.T) { + wires, txns := maxSizeTransferPool(t) + t.Logf("verified %d signed transactions at %d bytes each", len(wires), len(wires[0])) + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + for i := 0; i < 50; i++ { + entries, batchBytes, flushed := builder.Append(txns[i], len(wires[i])) + if i < 49 && flushed { + t.Fatalf("premature flush at %d", i) + } + if i == 49 { + if !flushed || len(entries) != 1 || len(entries[0].Txns) != 49 || batchBytes != 60424 { + t.Fatal("unexpected full-batch size") + } + } + } +} + +// 50k maximum-size transactions require two slots under the current 32,768 +// data-shred cap. Each iteration completes two slots of 25k transactions. +func BenchmarkProducer50kMaxSize(b *testing.B) { + const transactions = 50000 + const transactionsPerSlot = 25000 + wires, txns := maxSizeTransferPool(b) + leader := txfixture.PayerPrivateKey() + var lastBatches, lastPackets, lastMaxSlotDataShreds int + b.ReportAllocs() + b.ResetTimer() + for round := 0; round < b.N; round++ { + var parentID, parentRoot solana.Hash + totalBatches, totalPackets, maxSlotDataShreds := 0, 0, 0 + for block := 0; block < transactions/transactionsPerSlot; block++ { + slot := uint64(100 + block) + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + sink := &benchmarkPacketBroadcaster{} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: slot, ParentSlot: slot - 1, Version: 1, Broadcaster: sink, + ParentBlockID: parentID, ParentChainedMerkleRoot: parentRoot, + }) + if err := session.BroadcastHeader(parentID); err != nil { + b.Fatal(err) + } + for i := 0; i < transactionsPerSlot; i++ { + idx := (block*transactionsPerSlot + i) % len(txns) + entries, _, flushed := builder.Append(txns[idx], len(wires[idx])) + if flushed { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + totalBatches++ + } + } + if entries, _ := builder.Flush(); len(entries) > 0 { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + totalBatches++ + } + if err := session.BroadcastFooter(solana.Hash{1}, 0, nil, nil); err != nil { + b.Fatal(err) + } + if err := session.BroadcastEndingTickLast(builder.CurrentEntryHash()); err != nil { + b.Fatal(err) + } + parentID = session.BlockID(slot-1, parentID) + parentRoot = session.ChainedMerkleRoot() + // The generator emits equal numbers of data and coding shreds. + dataShreds := sink.packets / 2 + if dataShreds > costmodel.DefaultMaxDataShredsPerSlot { + b.Fatal("slot exceeds data-shred limit") + } + if dataShreds > maxSlotDataShreds { + maxSlotDataShreds = dataShreds + } + totalPackets += sink.packets + } + lastBatches, lastPackets, lastMaxSlotDataShreds = totalBatches, totalPackets, maxSlotDataShreds + } + b.StopTimer() + b.ReportMetric(transactions, "transactions/op") + b.ReportMetric(transactions/transactionsPerSlot, "slots/op") + b.ReportMetric(float64(len(wires[0])), "wire-B/tx") + b.ReportMetric(float64(lastBatches), "entry-batches/op") + b.ReportMetric(float64(lastPackets), "packets/op") + b.ReportMetric(float64(lastMaxSlotDataShreds), "max-data-shreds/slot") +} diff --git a/pkg/blockprod/readonly_block_bench_test.go b/pkg/blockprod/readonly_block_bench_test.go new file mode 100644 index 000000000..c605243f5 --- /dev/null +++ b/pkg/blockprod/readonly_block_bench_test.go @@ -0,0 +1,189 @@ +package blockprod + +import ( + "crypto/ed25519" + "crypto/sha256" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +const readonlyBlockAccepted = 48622 // 50M budget, including the next tx's upfront loaded-data reservation. +const readonlyActualCost = 1028 + +type readonlyBlockParent struct{ mem accounts.MemAccounts } + +func (p readonlyBlockParent) GetAccount(_ uint64, key solana.PublicKey) (*accounts.Account, error) { + raw := [32]byte(key) + return p.mem.GetAccount(&raw) +} + +type readonlyBlockFixture struct { + wires [][]byte + txs []*solana.Transaction + payers []solana.PublicKey + parent readonlyBlockParent +} + +func makeReadonlyBlockFixture(tb testing.TB, count int) readonlyBlockFixture { + tb.Helper() + f := readonlyBlockFixture{parent: readonlyBlockParent{accounts.NewMemAccounts()}} + keys := make([]ed25519.PrivateKey, 8) + for i := range keys { + seed := sha256.Sum256([]byte{byte(i), 73}) + keys[i] = ed25519.NewKeyFromSeed(seed[:]) + f.payers = append(f.payers, solana.PublicKeyFromBytes(keys[i].Public().(ed25519.PublicKey))) + } + pool := make([]solana.PublicKey, txfixture.ReadonlyPairPoolSize) + for i := range pool { + pool[i] = solana.PublicKey{byte(i + 1), 77} + require.NoError(tb, f.parent.mem.SetAccountWithoutLock(pool[i], &accounts.Account{ + Key: pool[i], Lamports: 1_000_000, Data: make([]byte, 9), Owner: solana.PublicKey{11}, RentEpoch: math.MaxUint64, + })) + } + for i := 0; i < count; i++ { + wire, err := txfixture.ReadonlyPairWire(keys[i%8], txfixture.TestBlockhash(), pool, i/8) + require.NoError(tb, err) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(tb, err) + f.wires = append(f.wires, wire) + f.txs = append(f.txs, tx) + } + return f +} + +func (f readonlyBlockFixture) bank(tb testing.TB, sink BatchSink) *TestEnv { + tb.Helper() + // Explicit active 200ms testnet budgets allow identical baseline/candidate + // benchmark fixtures; LimitsForSlot has separate epoch-transition tests. + limits := costmodel.DefaultLimits() + limits.BlockCost, limits.WritableAccountCost = 50_000_000, 20_000_000 + limits.AllocatedDataSizeDelta, limits.MaxEntryBytes = 50_000_000, 10*1024*1024-48 + env := NewTestEnv(TestEnvConfig{Limits: limits, Sink: sink}) + env.SlotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) + env.SlotCtx.UnrootedRead = f.parent + for _, payer := range f.payers { + require.NoError(tb, env.SlotCtx.Accounts.SetAccountWithoutLock(payer, &accounts.Account{ + Key: payer, Lamports: 10_000_000_000, Owner: addresses.SystemProgramAddr, RentEpoch: math.MaxUint64, + })) + } + return env +} + +// This is a capacity/correctness test, not a 200ms deadline assertion. It creates +// a full synthetic block, validates fee/cost accounting, and round-trips every +// emitted entry through the production shred generator and decoder. No network. +func TestReadonlyPairBlockCapacityAndShredRoundTrip(t *testing.T) { + f := makeReadonlyBlockFixture(t, readonlyBlockAccepted+1) + sink := &captureSink{} + env := f.bank(t, sink) + defer env.Close() + for i, tx := range f.txs { + result, reason := env.Bank.ForgeTransaction(tx, len(f.wires[i])) + if i < readonlyBlockAccepted { + require.Equal(t, ForgeAccepted, result, "transaction %d", i) + } else { + require.Equal(t, ForgeDroppedCost, result) + require.Equal(t, costmodel.ExceedBlockCost, reason) + } + } + env.Bank.Freeze() + require.Equal(t, uint64(readonlyBlockAccepted*readonlyActualCost), env.Bank.CostTracker().BlockCost()) + require.Equal(t, uint64(readonlyBlockAccepted), env.Bank.NumSignatures()) + require.Equal(t, uint64(readonlyBlockAccepted*5000), env.Bank.TxFeeAccumulator().TotalFees) + require.Len(t, env.Bank.ForgedTransactions(), readonlyBlockAccepted) + require.LessOrEqual(t, uint64(env.Bank.EntryBytes()), env.Bank.CostTracker().Limits().MaxEntryBytes) + for i, payer := range f.payers { + included := readonlyBlockAccepted / 8 + if i < readonlyBlockAccepted%8 { + included++ + } + acct, err := env.SlotCtx.GetAccount(payer) + require.NoError(t, err) + require.Equal(t, uint64(10_000_000_000-included*5000), acct.Lamports) + } + gen := turbine.ShredGenerator{Slot: 42, ParentSlot: 41, Version: 7} + var root solana.Hash + var nextData, nextCode uint32 + count, totalBytes := 0, 0 + previous := solana.Hash{0xab} + for i, entries := range sink.batches { + component, err := turbine.NewEntryBatch(entries) + require.NoError(t, err) + raw, err := turbine.MarshalBlockComponent(component) + require.NoError(t, err) + require.Equal(t, len(raw), sink.bytes[i]) + totalBytes += len(raw) + packets, chained, d, c, err := gen.MakeShredsFromData(txfixture.PayerPrivateKey(), raw, false, root, nextData, nextCode) + require.NoError(t, err) + root, nextData, nextCode = chained, d, c + var shreds []*turbine.Shred + for _, packet := range packets { + sh, err := turbine.ParseShred(packet) + require.NoError(t, err) + if sh.Type == turbine.ShredTypeData { + shreds = append(shreds, sh) + } + } + decoded, err := turbine.DecodeEntriesFromDataShreds(shreds) + require.NoError(t, err) + require.Equal(t, entries, decoded) + for _, entry := range decoded { + require.Equal(t, turbine.NextAlpenglowEntryHash(previous, entry.NumHashes, entry.Txns), entry.Hash) + previous = entry.Hash + for _, tx := range entry.Txns { + require.Equal(t, f.txs[count].Signatures, tx.Signatures) + count++ + } + } + } + require.Equal(t, readonlyBlockAccepted, count) + require.Equal(t, env.Bank.EntryBytes(), totalBytes) + require.Equal(t, env.Bank.EntryHash(), previous) + require.Less(t, nextData, uint32(16384)) // Leaves room for header/footer/ending tick. +} + +// One serial caller; signing and fixture/bank setup are excluded. Measures +// admission, execution, account publication, entry building and final flush. +// It excludes signature verification, actual AccountsDB, network and consensus. +func BenchmarkReadonlyPairFullBlock(b *testing.B) { + f := makeReadonlyBlockFixture(b, readonlyBlockAccepted) + for _, mode := range []string{"wire", "decoded"} { + b.Run(mode, func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + env := f.bank(b, nil) + b.StartTimer() + for j, wire := range f.wires { + var result ForgeResult + if mode == "wire" { + result, _ = env.Bank.Forge(wire) + } else { + result, _ = env.Bank.ForgeTransaction(f.txs[j], len(wire)) + } + if result != ForgeAccepted { + b.Fatalf("transaction %d: %v", j, result) + } + } + env.Bank.Freeze() + b.StopTimer() + if env.Bank.CostTracker().BlockCost() != readonlyBlockAccepted*readonlyActualCost { + b.Fatal("unexpected block cost") + } + env.Close() + b.StartTimer() + } + b.ReportMetric(float64(readonlyBlockAccepted), "tx/block") + }) + } +} diff --git a/pkg/blockprod/readonly_block_prepared_bench_test.go b/pkg/blockprod/readonly_block_prepared_bench_test.go new file mode 100644 index 000000000..68957fcef --- /dev/null +++ b/pkg/blockprod/readonly_block_prepared_bench_test.go @@ -0,0 +1,45 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/replay" +) + +// Models a queue already statically prepared before leadership. Preparation is +// shifted out of this timed bank phase, not eliminated from validator CPU work. +func BenchmarkReadonlyPairPreparedFullBlock(b *testing.B) { + f := makeReadonlyBlockFixture(b, readonlyBlockAccepted) + setup := f.bank(b, nil) + preparer := replay.NewTransactionPreparer(setup.SlotCtx.Features) + prepared := make([]*replay.PreparedTransaction, len(f.txs)) + for i, tx := range f.txs { + prepared[i] = preparer.Prepare(tx) + if prepared[i] == nil { + b.Fatal("preparation failed") + } + } + setup.Close() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.StopTimer() + env := f.bank(b, nil) + env.Bank.preparer = replay.NewTransactionPreparer(env.SlotCtx.Features) + b.StartTimer() + for j, tx := range f.txs { + result, _ := env.Bank.ForgePreparedTransaction(tx, len(f.wires[j]), prepared[j]) + if result != ForgeAccepted { + b.Fatalf("transaction %d: %v", j, result) + } + } + env.Bank.Freeze() + b.StopTimer() + if env.Bank.CostTracker().BlockCost() != readonlyBlockAccepted*readonlyActualCost { + b.Fatal("unexpected block cost") + } + env.Close() + b.StartTimer() + } + b.ReportMetric(float64(readonlyBlockAccepted), "tx/block") +} diff --git a/pkg/blockprod/readonly_load_bench_test.go b/pkg/blockprod/readonly_load_bench_test.go new file mode 100644 index 000000000..f5d24798d --- /dev/null +++ b/pkg/blockprod/readonly_load_bench_test.go @@ -0,0 +1,95 @@ +package blockprod + +import ( + "crypto/ed25519" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" +) + +type readonlyBenchParent struct{ mem accounts.MemAccounts } + +func (p readonlyBenchParent) GetAccount(_ uint64, key solana.PublicKey) (*accounts.Account, error) { + raw := [32]byte(key) + return p.mem.GetAccount(&raw) +} + +// Only bank admission/execution/entry building are timed. The immutable parent +// is in memory; this excludes actual AccountsDB, network and signing costs. +func BenchmarkReadonlyPairBank(b *testing.B) { + const perBank = 10000 + parent := readonlyBenchParent{accounts.NewMemAccounts()} + pool := make([]solana.PublicKey, 128) + for i := range pool { + pool[i] = solana.PublicKey{byte(i + 1), 77} + parent.mem.SetAccountWithoutLock(pool[i], &accounts.Account{Key: pool[i], Lamports: 1_000_000, Data: make([]byte, 9), Owner: solana.PublicKey{11}, RentEpoch: math.MaxUint64}) + } + txs := make([]*solana.Transaction, perBank) + key := txfixture.PayerPrivateKey() + payer := key.PublicKey() + hash := txfixture.TestBlockhash() + for i := range txs { + n := (i * 7919) % (128 * 127) + a, c := n/127, n%127 + if c >= a { + c++ + } + msg := []byte{1, 0, 2, 3} + msg = append(msg, payer[:]...) + msg = append(msg, pool[a][:]...) + msg = append(msg, pool[c][:]...) + msg = append(msg, hash[:]...) + msg = append(msg, 0) + wire := []byte{1} + wire = append(wire, ed25519.Sign(ed25519.PrivateKey(key), msg)...) + wire = append(wire, msg...) + var err error + txs[i], err = solana.TransactionFromBytes(wire) + if err != nil { + b.Fatal(err) + } + } + var env *TestEnv + var prepared []*replay.PreparedTransaction + defer func() { + if env != nil { + env.Close() + } + }() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i%perBank == 0 { + b.StopTimer() + if env != nil { + env.Close() + } + env = NewTestEnv(TestEnvConfig{}) + env.SlotCtx.Features.EnableFeature(features.RemoveAccountsDeltaHash, 0) + env.SlotCtx.UnrootedRead = parent + env.Bank.preparer = replay.NewTransactionPreparer(env.SlotCtx.Features) + if prepared == nil { + prepared = make([]*replay.PreparedTransaction, perBank) + for j, t := range txs { + prepared[j] = env.Bank.preparer.Prepare(t) + if prepared[j] == nil { + b.Fatal("preparation failed") + } + } + } + b.StartTimer() + } + outcome, reason := env.Bank.ForgePreparedTransaction(txs[i%perBank], 198, prepared[i%perBank]) + if outcome != ForgeAccepted { + b.Fatalf("%v / %v", outcome, reason) + } + if i%perBank == perBank-1 && env.Bank.CostTracker().BlockCost() != perBank*1028 { + b.Fatalf("unexpected cost %d", env.Bank.CostTracker().BlockCost()) + } + } +} diff --git a/pkg/blockprod/scheduler/buffer.go b/pkg/blockprod/scheduler/buffer.go index 307dc0b28..2aa5a70c5 100644 --- a/pkg/blockprod/scheduler/buffer.go +++ b/pkg/blockprod/scheduler/buffer.go @@ -4,15 +4,17 @@ import ( "container/heap" "sync" + "github.com/Overclock-Validator/mithril/pkg/replay" "github.com/gagliardetto/solana-go" ) -// MaxBufferedTxns is the hard cap on cross-slot buffered transactions. +// MaxBufferedTxns is the default cap on cross-slot buffered transactions. const MaxBufferedTxns = 2 * 65536 // entry is one buffered, scored transaction. type entry struct { - tx *solana.Transaction + tx *solana.Transaction + prepared *replay.PreparedTransaction // wire is an owned copy of the packet bytes. Parsed tx fields may alias it // (solana-go decoder slices), so it must outlive any use of tx. wire []byte @@ -25,37 +27,34 @@ type entry struct { // (e.g. cost limit). The entry is retained for cross-slot retry. skipGen uint64 - alive bool - maxIdx int - minIdx int + alive bool + // Indexes belong to Buffer.mu. Every buffered entry appears exactly once + // in each heap; -1 denotes absence while an entry is owned by the consumer. + maxIndex, minIndex int } type maxHeap []*entry func (h maxHeap) Len() int { return len(h) } func (h maxHeap) Less(i, j int) bool { - if h[i].reward != h[j].reward { - return h[i].reward > h[j].reward - } - return h[i].seq < h[j].seq // older first on ties + return higherPriority(h[i], h[j]) } func (h maxHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] - h[i].maxIdx = i - h[j].maxIdx = j + h[i].maxIndex, h[j].maxIndex = i, j } func (h *maxHeap) Push(x any) { e := x.(*entry) - e.maxIdx = len(*h) + e.maxIndex = len(*h) *h = append(*h, e) } func (h *maxHeap) Pop() any { old := *h n := len(old) e := old[n-1] + e.maxIndex = -1 old[n-1] = nil *h = old[:n-1] - e.maxIdx = -1 return e } @@ -71,21 +70,20 @@ func (h minHeap) Less(i, j int) bool { } func (h minHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] - h[i].minIdx = i - h[j].minIdx = j + h[i].minIndex, h[j].minIndex = i, j } func (h *minHeap) Push(x any) { e := x.(*entry) - e.minIdx = len(*h) + e.minIndex = len(*h) *h = append(*h, e) } func (h *minHeap) Pop() any { old := *h n := len(old) e := old[n-1] + e.minIndex = -1 old[n-1] = nil *h = old[:n-1] - e.minIdx = -1 return e } @@ -131,27 +129,41 @@ const ( InsertRejectedCapacity ) -// Insert adds e when not a duplicate. At capacity, the lowest-reward entry is -// evicted if e has a strictly higher reward; otherwise e is rejected. -func (b *Buffer) Insert(e *entry) (InsertResult, *entry) { - if e == nil || e.tx == nil { - return InsertRejectedCapacity, nil - } +// precheck rejects entries that cannot be admitted now, without reserving space +// or evicting anything. Preparation runs outside mu; Insert must recheck because +// concurrent arrivals or draining can change both duplicates and the floor. +func (b *Buffer) precheck(e *entry) InsertResult { b.mu.Lock() defer b.mu.Unlock() + return b.admissionLocked(e) +} +func (b *Buffer) admissionLocked(e *entry) InsertResult { + if e == nil || e.tx == nil { + return InsertRejectedCapacity + } if _, exists := b.byHash[e.messageHash]; exists { - return InsertDuplicate, nil + return InsertDuplicate } - var evicted *entry if b.alive >= b.capacity { min := b.peekMinAliveLocked() - if min == nil { - return InsertRejectedCapacity, nil - } - if e.reward <= min.reward { - return InsertRejectedCapacity, nil + if min == nil || e.reward <= min.reward { + return InsertRejectedCapacity } + } + return InsertAccepted +} + +// Insert adds e when not a duplicate. At capacity, the lowest-reward entry is +// evicted if e has a strictly higher reward; otherwise e is rejected. +func (b *Buffer) Insert(e *entry) (InsertResult, *entry) { + b.mu.Lock() + defer b.mu.Unlock() + if result := b.admissionLocked(e); result != InsertAccepted { + return result, nil + } + var evicted *entry + if b.alive >= b.capacity { evicted = b.popMinAliveLocked() } b.pushAliveLocked(e) @@ -173,21 +185,19 @@ func (b *Buffer) Cleanup(drop func(*entry) bool) int { b.mu.Lock() defer b.mu.Unlock() - var doomed []*entry + dropped := 0 for _, e := range b.byHash { - if e.alive && drop(e) { - doomed = append(doomed, e) + if drop(e) { + b.killLocked(e) + dropped++ } } - for _, e := range doomed { - b.killLocked(e) - } - b.drainDeadLocked() - return len(doomed) + return dropped } func (b *Buffer) pushAliveLocked(e *entry) { e.alive = true + e.maxIndex, e.minIndex = -1, -1 b.byHash[e.messageHash] = e heap.Push(&b.max, e) heap.Push(&b.min, e) @@ -198,56 +208,79 @@ func (b *Buffer) killLocked(e *entry) { if e == nil || !e.alive { return } + if e.maxIndex >= 0 { + heap.Remove(&b.max, e.maxIndex) + } + if e.minIndex >= 0 { + heap.Remove(&b.min, e.minIndex) + } e.alive = false delete(b.byHash, e.messageHash) b.alive-- } func (b *Buffer) popMaxAliveLocked() *entry { - for b.max.Len() > 0 { - e := heap.Pop(&b.max).(*entry) - if !e.alive { - continue - } + if b.max.Len() > 0 { + e := b.max.popEntry() b.killLocked(e) return e } return nil } -func (b *Buffer) peekMinAliveLocked() *entry { - for b.min.Len() > 0 { - if b.min[0].alive { - return b.min[0] +// popEntry moves the winning child into the hole at each level. This avoids +// interface dispatch and swapping two entries at every level of a large queue. +// Ordering is identical to maxHeap.Less, including FIFO for equal rewards. +func (h *maxHeap) popEntry() *entry { + nodes := *h + root := nodes[0] + root.maxIndex = -1 + last := nodes[len(nodes)-1] + nodes[len(nodes)-1] = nil + nodes = nodes[:len(nodes)-1] + if len(nodes) > 0 { + i := 0 + for { + child := 2*i + 1 + if child >= len(nodes) { + break + } + if child+1 < len(nodes) && higherPriority(nodes[child+1], nodes[child]) { + child++ + } + if !higherPriority(nodes[child], last) { + break + } + nodes[i] = nodes[child] + nodes[i].maxIndex = i + i = child } - heap.Pop(&b.min) + nodes[i] = last + last.maxIndex = i + } + *h = nodes + return root +} + +func higherPriority(a, b *entry) bool { + if a.reward != b.reward { + return a.reward > b.reward + } + return a.seq < b.seq +} + +func (b *Buffer) peekMinAliveLocked() *entry { + if b.min.Len() > 0 { + return b.min[0] } return nil } func (b *Buffer) popMinAliveLocked() *entry { - for b.min.Len() > 0 { + if b.min.Len() > 0 { e := heap.Pop(&b.min).(*entry) - if !e.alive { - continue - } b.killLocked(e) return e } return nil } - -func (b *Buffer) drainDeadLocked() { - for b.max.Len() > 0 { - if b.max[0].alive { - break - } - heap.Pop(&b.max) - } - for b.min.Len() > 0 { - if b.min[0].alive { - break - } - heap.Pop(&b.min) - } -} diff --git a/pkg/blockprod/scheduler/buffer_bench_test.go b/pkg/blockprod/scheduler/buffer_bench_test.go new file mode 100644 index 000000000..ccdaa3af0 --- /dev/null +++ b/pkg/blockprod/scheduler/buffer_bench_test.go @@ -0,0 +1,45 @@ +package scheduler + +import ( + "encoding/binary" + "testing" + + "github.com/gagliardetto/solana-go" +) + +// BenchmarkBufferDrain measures selection from a large, already-filled TPU +// queue. Transaction decoding and queue filling are outside the timed region. +func BenchmarkBufferDrain(b *testing.B) { + const count = 120000 + for _, mixed := range []bool{false, true} { + name := "equal_rewards" + if mixed { + name = "mixed_rewards" + } + b.Run(name, func(b *testing.B) { + var buffer *Buffer + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if i%count == 0 { + b.StopTimer() + buffer = NewBuffer(count) + for j := 0; j < count; j++ { + e := &entry{tx: &solana.Transaction{}, seq: uint64(j), reward: 2500} + binary.LittleEndian.PutUint64(e.messageHash[:], uint64(j)) + if mixed { + e.reward = uint64(j*7919) % 1000 + } + if result, _ := buffer.Insert(e); result != InsertAccepted { + b.Fatal("queue fill failed") + } + } + b.StartTimer() + } + if buffer.PopMax() == nil { + b.Fatal("queue drained prematurely") + } + } + }) + } +} diff --git a/pkg/blockprod/scheduler/buffer_order_test.go b/pkg/blockprod/scheduler/buffer_order_test.go new file mode 100644 index 000000000..11a091bfb --- /dev/null +++ b/pkg/blockprod/scheduler/buffer_order_test.go @@ -0,0 +1,106 @@ +package scheduler + +import ( + "container/heap" + "encoding/binary" + "math/rand" + "sort" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestMaxHeapRemovalMatchesSortedOrder(t *testing.T) { + rng := rand.New(rand.NewSource(417)) + for _, count := range []int{1, 2, 3, 4, 7, 8, 9, 127, 128, 129, 4095, 4096, 4097} { + var h maxHeap + want := make([]*entry, count) + for i := range want { + want[i] = &entry{seq: uint64(i), reward: uint64(rng.Intn(17))} + heap.Push(&h, want[i]) + } + sort.Slice(want, func(i, j int) bool { + if want[i].reward == want[j].reward { + return want[i].seq < want[j].seq + } + return want[i].reward > want[j].reward + }) + backing := h[:cap(h)] + for i, expected := range want { + require.Same(t, expected, h.popEntry(), "count=%d pop=%d", count, i) + require.Nil(t, backing[len(h)], "removed pointer retained") + } + } +} + +// Compare interleaved insertion, eviction, cleanup and removal with a small +// unsorted reference model. Reusing hashes after removal exercises replacement +// entries as well as counterpart removal from both indexed heaps. +func TestBufferMixedOperationsMatchReference(t *testing.T) { + const capacity = 64 + rng := rand.New(rand.NewSource(418)) + b := NewBuffer(capacity) + model := make(map[[32]byte]*entry) + best := func(high bool) *entry { + var found *entry + for _, e := range model { + if found == nil || (high && (e.reward > found.reward || e.reward == found.reward && e.seq < found.seq)) || + (!high && (e.reward < found.reward || e.reward == found.reward && e.seq > found.seq)) { + found = e + } + } + return found + } + for step := 0; step < 10000; step++ { + switch action := rng.Intn(10); { + case action < 7: + e := &entry{tx: &solana.Transaction{}, seq: uint64(step), reward: uint64(rng.Intn(16))} + binary.LittleEndian.PutUint64(e.messageHash[:], uint64(rng.Intn(256))) + wantResult := InsertAccepted + var wantEvicted *entry + if _, duplicate := model[e.messageHash]; duplicate { + wantResult = InsertDuplicate + } else if len(model) == capacity { + lowest := best(false) + if e.reward <= lowest.reward { + wantResult = InsertRejectedCapacity + } else { + wantEvicted = lowest + delete(model, lowest.messageHash) + } + } + if wantResult == InsertAccepted { + model[e.messageHash] = e + } + got, evicted := b.Insert(e) + require.Equal(t, wantResult, got, "step=%d", step) + require.True(t, wantEvicted == evicted, "eviction differs at step=%d", step) + case action < 9: + want := best(true) + got := b.PopMax() + require.True(t, want == got, "selection differs at step=%d", step) + if want != nil { + delete(model, want.messageHash) + } + default: + mod := uint64(rng.Intn(11)) + want := 0 + for hash, e := range model { + if e.seq%11 == mod { + delete(model, hash) + want++ + } + } + require.Equal(t, want, b.Cleanup(func(e *entry) bool { return e.seq%11 == mod })) + } + require.Equal(t, len(model), b.Len(), "step=%d", step) + assertBufferIndexes(t, b) + } + for len(model) > 0 { + want := best(true) + require.Same(t, want, b.PopMax()) + delete(model, want.messageHash) + } + require.Nil(t, b.PopMax()) +} diff --git a/pkg/blockprod/scheduler/buffer_retention_test.go b/pkg/blockprod/scheduler/buffer_retention_test.go new file mode 100644 index 000000000..6b24aac8a --- /dev/null +++ b/pkg/blockprod/scheduler/buffer_retention_test.go @@ -0,0 +1,158 @@ +package scheduler + +import ( + "encoding/binary" + "sync" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func retainedTestEntry(id, reward uint64) *entry { + e := &entry{tx: &solana.Transaction{}, wire: []byte{1, 2, 3}, seq: id, reward: reward} + binary.LittleEndian.PutUint64(e.messageHash[:], id) + return e +} + +// Check both membership and backing-array references: shrinking a slice alone +// must not leave transaction payloads reachable outside its visible length. +func assertBufferIndexes(t *testing.T, b *Buffer) { + t.Helper() + b.mu.Lock() + defer b.mu.Unlock() + require.Equal(t, b.alive, len(b.byHash)) + require.Equal(t, b.alive, len(b.max)) + require.Equal(t, b.alive, len(b.min)) + require.LessOrEqual(t, b.alive, b.capacity) + for i, e := range b.max { + require.True(t, e.alive) + require.Equal(t, i, e.maxIndex) + require.Same(t, e, b.byHash[e.messageHash]) + require.Same(t, e, b.min[e.minIndex]) + if i > 0 { + require.False(t, b.max.Less(i, (i-1)/2)) + } + } + for i, e := range b.min { + require.Equal(t, i, e.minIndex) + require.Same(t, e, b.max[e.maxIndex]) + if i > 0 { + require.False(t, b.min.Less(i, (i-1)/2)) + } + } + for _, backing := range [][]*entry{b.max[:cap(b.max)], b.min[:cap(b.min)]} { + for _, e := range backing[b.alive:] { + require.Nil(t, e) + } + } +} + +func TestBufferConsumedEntriesReleaseBothHeapReferences(t *testing.T) { + b := NewBuffer(256) + pinned := retainedTestEntry(0, 1) + b.Insert(pinned) + for i := uint64(1); i <= 100000; i++ { + e := retainedTestEntry(i, 2) + result, _ := b.Insert(e) + require.Equal(t, InsertAccepted, result) + require.Same(t, e, b.PopMax()) + // The consumer still owns usable payloads after index removal. + require.NotNil(t, e.tx) + require.Equal(t, []byte{1, 2, 3}, e.wire) + if i%1000 == 0 { + assertBufferIndexes(t, b) + } + } + b.Cleanup(func(*entry) bool { return false }) + assertBufferIndexes(t, b) + t.Logf("capacity=%d active=%d max_refs=%d min_refs=%d", b.capacity, b.Len(), len(b.max), len(b.min)) + require.Same(t, pinned, b.PopMax()) + assertBufferIndexes(t, b) +} + +func TestBufferEvictionAndCleanupReleaseBothHeapReferences(t *testing.T) { + b := NewBuffer(2) + pinned := retainedTestEntry(0, 1000000) + b.Insert(pinned) + previous := retainedTestEntry(1, 1) + b.Insert(previous) + for i := uint64(2); i < 10000; i++ { + next := retainedTestEntry(i, i) + result, evicted := b.Insert(next) + require.Equal(t, InsertAccepted, result) + require.Same(t, previous, evicted) + require.False(t, evicted.alive) + require.Equal(t, -1, evicted.maxIndex) + require.Equal(t, -1, evicted.minIndex) + previous = next + if i%100 == 0 { + assertBufferIndexes(t, b) + } + } + require.Equal(t, 1, b.Cleanup(func(e *entry) bool { return e != pinned })) + assertBufferIndexes(t, b) + require.Same(t, pinned, b.PopMax()) + assertBufferIndexes(t, b) +} + +func TestBufferRepeatedRebufferPreservesNewHigherPriorityArrivals(t *testing.T) { + s := New(nil) + s.bankGen = 1 + skipped := retainedTestEntry(1, 10) + skipped.skipGen = s.bankGen + s.buffer.Insert(skipped) + for i := uint64(2); i < 10002; i++ { + low := retainedTestEntry(i, 1) + s.buffer.Insert(low) + picked, retry := s.popSchedulable(s.bankGen) + require.Same(t, low, picked) + require.Equal(t, []*entry{skipped}, retry) + for _, e := range retry { + s.rebuffer(e) + } + if i%100 == 0 { + assertBufferIndexes(t, s.buffer) + } + } + // Preserve the existing scan/retry policy when a higher-fee packet arrives. + high := retainedTestEntry(20000, 20) + s.buffer.Insert(high) + picked, retry := s.popSchedulable(s.bankGen) + require.Same(t, high, picked) + require.Empty(t, retry) + // A later bank may retry the previously skipped transaction. + s.bankGen++ + picked, retry = s.popSchedulable(s.bankGen) + require.Same(t, skipped, picked) + require.Empty(t, retry) + assertBufferIndexes(t, s.buffer) +} + +func TestBufferConcurrentInsertRemovalAndCleanup(t *testing.T) { + b := NewBuffer(64) + var workers sync.WaitGroup + for worker := uint64(0); worker < 4; worker++ { + workers.Go(func() { + for i := uint64(0); i < 2000; i++ { + id := worker*2000 + i + b.Insert(retainedTestEntry(id, id%17)) + } + }) + } + workers.Go(func() { + for i := 0; i < 8000; i++ { + b.PopMax() + } + }) + workers.Go(func() { + for i := 0; i < 100; i++ { + b.Cleanup(func(e *entry) bool { return e.seq%3 == 0 }) + } + }) + workers.Wait() + assertBufferIndexes(t, b) + for b.PopMax() != nil { + } + assertBufferIndexes(t, b) +} diff --git a/pkg/blockprod/scheduler/buffer_test.go b/pkg/blockprod/scheduler/buffer_test.go index 9b85554ce..6ec46415d 100644 --- a/pkg/blockprod/scheduler/buffer_test.go +++ b/pkg/blockprod/scheduler/buffer_test.go @@ -79,3 +79,28 @@ func TestBufferCleanup(t *testing.T) { require.Equal(t, 1, b.Len()) require.Equal(t, byte(2), b.PopMax().messageHash[0]) } + +func TestBufferPrecheckDoesNotReserveAndInsertRechecks(t *testing.T) { + b := NewBuffer(1) + first := testEntry(10, 1, 1) + require.Equal(t, InsertAccepted, b.precheck(first)) + require.Zero(t, b.Len()) + b.Insert(first) + require.Equal(t, InsertDuplicate, b.precheck(testEntry(20, 2, 1))) + require.Equal(t, InsertRejectedCapacity, b.precheck(testEntry(10, 2, 2))) + candidate := testEntry(20, 3, 3) + require.Equal(t, InsertAccepted, b.precheck(candidate)) + // A precheck must not evict the existing entry, and a later higher-priority + // arrival can reject the candidate even though its precheck succeeded. + require.Equal(t, first, b.PopMax()) + b.Insert(testEntry(30, 4, 4)) + result, evicted := b.Insert(candidate) + require.Equal(t, InsertRejectedCapacity, result) + require.Nil(t, evicted) + b.PopMax() + require.Equal(t, InsertAccepted, b.precheck(candidate)) + b.Insert(testEntry(20, 5, 3)) + result, evicted = b.Insert(candidate) + require.Equal(t, InsertDuplicate, result) + require.Nil(t, evicted) +} diff --git a/pkg/blockprod/scheduler/config_test.go b/pkg/blockprod/scheduler/config_test.go new file mode 100644 index 000000000..9f43ed511 --- /dev/null +++ b/pkg/blockprod/scheduler/config_test.go @@ -0,0 +1,28 @@ +package scheduler + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/stretchr/testify/require" +) + +func TestConfiguredQueueCapacityAndPreparation(t *testing.T) { + defaults := NewWithConfig(nil, Config{}) + require.Equal(t, MaxBufferedTxns, defaults.buffer.Capacity()) + feats := features.NewFeaturesDefault() + custom := NewWithConfig(nil, Config{MaxBufferedTransactions: 2, FeatureSource: func() *features.Features { return feats }}) + require.Equal(t, 2, custom.buffer.Capacity()) + require.NotNil(t, custom.preparer.Load()) + for i := byte(1); i <= 2; i++ { + result, _ := custom.buffer.Insert(testEntry(10, uint64(i), i)) + require.Equal(t, InsertAccepted, result) + } + result, evicted := custom.buffer.Insert(testEntry(9, 3, 3)) + require.Equal(t, InsertRejectedCapacity, result) + require.Nil(t, evicted) + result, evicted = custom.buffer.Insert(testEntry(11, 4, 4)) + require.Equal(t, InsertAccepted, result) + require.Equal(t, uint64(2), evicted.seq) + require.Equal(t, 2, custom.Buffered()) +} diff --git a/pkg/blockprod/scheduler/scheduler.go b/pkg/blockprod/scheduler/scheduler.go index de6196f30..9b0864d27 100644 --- a/pkg/blockprod/scheduler/scheduler.go +++ b/pkg/blockprod/scheduler/scheduler.go @@ -9,6 +9,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/blockprod" "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/replay" "github.com/Overclock-Validator/mithril/pkg/tpu/packet" "github.com/gagliardetto/solana-go" ) @@ -43,9 +44,12 @@ type Stats struct { // Scheduler buffers verified TPU transactions in a reward-ordered heap and // drains into the active WorkingBank when one is published. type Scheduler struct { - banks BankSource - feats *features.Features - buffer *Buffer + banks BankSource + feats *features.Features + buffer *Buffer + preparer atomic.Pointer[replay.TransactionPreparer] + featureSource func() *features.Features + lastPreparationRefresh time.Time seq atomic.Uint64 wake chan struct{} @@ -80,6 +84,37 @@ func New(banks BankSource) *Scheduler { } } +// NewWithFeatureSource prepares queued messages against a replay-tip snapshot. +// The active bank independently verifies feature compatibility before reuse. +func NewWithFeatureSource(banks BankSource, source func() *features.Features) *Scheduler { + return NewWithConfig(banks, Config{FeatureSource: source}) +} + +// Config controls the bounded TPU queue and static preparation source. +type Config struct { + // MaxBufferedTransactions defaults to MaxBufferedTxns when zero. + MaxBufferedTransactions int + FeatureSource func() *features.Features +} + +func NewWithConfig(banks BankSource, cfg Config) *Scheduler { + s := New(banks) + if cfg.MaxBufferedTransactions > 0 { + s.buffer = NewBuffer(cfg.MaxBufferedTransactions) + } + s.featureSource = cfg.FeatureSource + s.refreshPreparation() + return s +} + +func (s *Scheduler) refreshPreparation() { + if s.featureSource == nil || time.Since(s.lastPreparationRefresh) < 100*time.Millisecond { + return + } + s.preparer.Store(replay.NewTransactionPreparer(s.featureSource())) + s.lastPreparationRefresh = time.Now() +} + // Start launches the bank-gated drain loop. func (s *Scheduler) Start(ctx context.Context) { s.startOnce.Do(func() { @@ -145,7 +180,12 @@ func (s *Scheduler) Receive(pkt packet.Packet) { reward: reward, seq: s.seq.Add(1), } - result, evicted := s.buffer.Insert(e) + result := s.buffer.precheck(e) + var evicted *entry + if result == InsertAccepted { + e.prepared = s.preparer.Load().Prepare(tx) + result, evicted = s.buffer.Insert(e) + } s.mu.Lock() switch result { case InsertAccepted: @@ -219,6 +259,7 @@ func (s *Scheduler) drainLoop(ctx context.Context) { if ctx.Err() != nil { return } + s.refreshPreparation() bank := s.banks.WorkingBank() s.noteBank(bank) if bank == nil { @@ -258,15 +299,10 @@ func (s *Scheduler) drainLoop(ctx context.Context) { continue } - // Prefer the owned wire so forge reparses from stable bytes even if the - // retained tx view was somehow mutated after buffering. - var result blockprod.ForgeResult - var reason costmodel.ExceedReason - if len(e.wire) > 0 { - result, reason = bank.Forge(e.wire) - } else { - result, reason = bank.ForgeTransaction(e.tx, e.wireSize) - } + // Receive owns the wire and its decoded transaction for the entire queue + // lifetime. Execution modifies transaction-local account clones, not + // this immutable message, so reuse the decoded transaction across banks. + result, reason := bank.ForgePreparedTransaction(e.tx, e.wireSize, e.prepared) switch result { case blockprod.ForgeDroppedNoLeader: s.rebuffer(e) diff --git a/pkg/blockprod/scheduler/scheduler_test.go b/pkg/blockprod/scheduler/scheduler_test.go index e4b74e092..f83711312 100644 --- a/pkg/blockprod/scheduler/scheduler_test.go +++ b/pkg/blockprod/scheduler/scheduler_test.go @@ -7,6 +7,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/blockprod" "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/fees" "github.com/Overclock-Validator/mithril/pkg/tpu/packet" "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" @@ -244,7 +245,9 @@ func TestMaxBufferedTxnsConstant(t *testing.T) { } func TestSchedulerCopiesPooledPacketBytes(t *testing.T) { - sched := New(blockprod.NewController()) + env := blockprod.NewTestEnv(blockprod.TestEnvConfig{}) + defer env.Close() + sched := NewWithFeatureSource(blockprod.NewController(), func() *features.Features { return env.SlotCtx.Features.Clone() }) pool := packet.NewPool(1) buf, idx, ok := pool.Acquire() require.True(t, ok) @@ -269,11 +272,26 @@ func TestSchedulerCopiesPooledPacketBytes(t *testing.T) { e := sched.buffer.PopMax() require.NotNil(t, e) require.Equal(t, wire, e.wire) + require.NotNil(t, e.prepared) // Re-parse and verify the buffered transaction still has a valid signature. tx, err := solana.TransactionFromBytes(e.wire) require.NoError(t, err) require.Equal(t, e.tx.Signatures[0], tx.Signatures[0]) + + // Exercise the actual drain path using the retained decoded transaction, + // after its original pooled packet has already been overwritten. + res, _ := sched.buffer.Insert(e) + require.Equal(t, InsertAccepted, res) + sched.banks = env.Controller + sched.Start(context.Background()) + defer sched.Stop() + require.Eventually(t, func() bool { return sched.Stats().Accepted == 1 }, time.Second, time.Millisecond) + forged := env.Bank.ForgedTransactions() + require.Len(t, forged, 1) + forgedWire, err := forged[0].MarshalBinary() + require.NoError(t, err) + require.Equal(t, wire, forgedWire) } func TestClassifyBufferedExpired(t *testing.T) { diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e7416c90..d37fb3c0a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -235,6 +235,8 @@ type ValidatorConfig struct { TPUQUICBindAddr string `toml:"tpu_quic_bind_addr" mapstructure:"tpu_quic_bind_addr"` AdvertisedIP string `toml:"advertised_ip" mapstructure:"advertised_ip"` TPUSigverifyWorkers int `toml:"tpu_sigverify_workers" mapstructure:"tpu_sigverify_workers"` + BlockCompletionReserveMs int `toml:"block_completion_reserve_ms" mapstructure:"block_completion_reserve_ms"` + TPUMaxBufferedTransactions int `toml:"tpu_max_buffered_transactions" mapstructure:"tpu_max_buffered_transactions"` } // Config holds all configuration options for Mithril (Firedancer-style hierarchy) diff --git a/pkg/costmodel/entry_bytes.go b/pkg/costmodel/entry_bytes.go index ab91b8d5a..7cfec6481 100644 --- a/pkg/costmodel/entry_bytes.go +++ b/pkg/costmodel/entry_bytes.go @@ -28,8 +28,11 @@ func PackEntryBytesMax(slotMaxDataShreds, maxMicroblock uint64) uint64 { // DefaultPackEntryBytes is min(shred-safe, SIMD-0525) minus one ending tick. func DefaultPackEntryBytes() uint64 { - shredSafe := PackEntryBytesMax(DefaultMaxDataShredsPerSlot, MaxMicroblockBytes) - cap := uint64(DefaultMaxEntryBytesPerSlot) + return packEntryBytes(DefaultMaxDataShredsPerSlot, DefaultMaxEntryBytesPerSlot) +} + +func packEntryBytes(maxDataShreds, cap uint64) uint64 { + shredSafe := PackEntryBytesMax(maxDataShreds, MaxMicroblockBytes) if shredSafe > 0 && shredSafe < cap { cap = shredSafe } diff --git a/pkg/costmodel/limits.go b/pkg/costmodel/limits.go index 2e4766569..26091a3d0 100644 --- a/pkg/costmodel/limits.go +++ b/pkg/costmodel/limits.go @@ -1,7 +1,11 @@ package costmodel import ( + "fmt" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/safemath" + "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/gagliardetto/solana-go" ) @@ -47,11 +51,11 @@ const ( TypicalDataShredPayloadBytes = 963 // DataShredsPerFECSet matches turbine's 32:32 erasure batch. DataShredsPerFECSet = 32 - // FECSetsPerBatch is the close watermark: hold until one FEC set is full. - FECSetsPerBatch = 1 + // FECSetsPerBatch is the close watermark: hold until two FEC sets are full. + FECSetsPerBatch = 2 // TypicalFECSetPayloadBytes is one full unsigned FEC set. TypicalFECSetPayloadBytes = DataShredsPerFECSet * TypicalDataShredPayloadBytes - // DefaultTargetBatchBytes is one FEC set. A short leftover is only + // DefaultTargetBatchBytes is two FEC sets. A short leftover is only // emitted at slot end (Freeze / ending tick). DefaultTargetBatchBytes = FECSetsPerBatch * TypicalFECSetPayloadBytes ) @@ -75,11 +79,54 @@ func DefaultLimits() Limits { } } -// LimitsForFeatures returns the cost limits selected by the bank's feature set. +// LimitsForFeatures returns the legacy 400ms budgets. Live banks must use +// LimitsForSlot to apply slot-time reductions at the correct epoch boundary. func LimitsForFeatures(feats *features.Features) Limits { limits := DefaultLimits() if feats != nil && feats.IsActive(features.RaiseBlockLimitsTo100m) { limits.BlockCost = MaxBlockUnitsSIMD0286 + limits.WritableAccountCost = 40_000_000 } return limits } + +// LimitsForSlot mirrors Agave v4.3.0-rc.1 runtime/src/slot_params.rs. +// Reference: https://github.com/anza-xyz/agave/blob/v4.3.0-rc.1/runtime/src/slot_params.rs A slot-time +// gate takes effect in the epoch after activation; among effective gates the +// shortest duration wins, even if longer-duration gates activate later. +func LimitsForSlot(feats *features.Features, schedule *sealevel.SysvarEpochSchedule, slot uint64) (Limits, error) { + limits := DefaultLimits() + for _, transition := range []struct { + gate features.FeatureGate + account, block, data, shreds, entries uint64 + }{ + {features.ReduceSlotTimeTo350ms, 21_000_000, 52_500_000, 87_500_000, 28_672, 18_350_080}, + {features.ReduceSlotTimeTo300ms, 18_000_000, 45_000_000, 75_000_000, 24_576, 15_728_640}, + {features.ReduceSlotTimeTo250ms, 15_000_000, 37_500_000, 62_500_000, 20_480, 13_107_200}, + {features.ReduceSlotTimeTo200ms, 12_000_000, 30_000_000, 50_000_000, 16_384, 10_485_760}, + } { + if feats == nil { + break + } + activation, active := feats.ActivationSlot(transition.gate) + if !active { + continue + } + if schedule == nil || schedule.SlotsPerEpoch == 0 { + return Limits{}, fmt.Errorf("epoch schedule required for slot-time cost limits") + } + effective := schedule.FirstSlotInEpoch(safemath.SaturatingAddU64(schedule.GetEpoch(activation), 1)) + if effective > slot { + continue + } + limits.WritableAccountCost = transition.account + limits.BlockCost = transition.block + limits.AllocatedDataSizeDelta = transition.data + limits.MaxEntryBytes = packEntryBytes(transition.shreds, transition.entries) + } + if feats != nil && feats.IsActive(features.RaiseBlockLimitsTo100m) { + limits.BlockCost = limits.BlockCost * 100 / 60 + limits.WritableAccountCost = limits.WritableAccountCost * 100 / 60 + } + return limits, nil +} diff --git a/pkg/costmodel/limits_test.go b/pkg/costmodel/limits_test.go new file mode 100644 index 000000000..60de4062a --- /dev/null +++ b/pkg/costmodel/limits_test.go @@ -0,0 +1,13 @@ +package costmodel + +import "testing" + +func TestDefaultTargetBatchBytesMatchesTwoTypicalFECSets(t *testing.T) { + const want = 61_632 + if DefaultTargetBatchBytes != want { + t.Fatalf("DefaultTargetBatchBytes = %d, want %d", DefaultTargetBatchBytes, want) + } + if DefaultTargetBatchBytes != 2*DataShredsPerFECSet*TypicalDataShredPayloadBytes { + t.Fatal("default batch target must remain two complete typical FEC payloads") + } +} diff --git a/pkg/costmodel/slot_limits_test.go b/pkg/costmodel/slot_limits_test.go new file mode 100644 index 000000000..0d521e744 --- /dev/null +++ b/pkg/costmodel/slot_limits_test.go @@ -0,0 +1,88 @@ +package costmodel + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/stretchr/testify/require" +) + +func TestSlotLimitsMatchAgaveTable(t *testing.T) { + schedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100} + for _, tc := range []struct { + name string + gate features.FeatureGate + account, block, data, shreds, entries uint64 + }{ + {"400ms", features.FeatureGate{}, 24_000_000, 60_000_000, 100_000_000, 32768, 20 * 1024 * 1024}, + {"350ms", features.ReduceSlotTimeTo350ms, 21_000_000, 52_500_000, 87_500_000, 28672, 18_350_080}, + {"300ms", features.ReduceSlotTimeTo300ms, 18_000_000, 45_000_000, 75_000_000, 24576, 15_728_640}, + {"250ms", features.ReduceSlotTimeTo250ms, 15_000_000, 37_500_000, 62_500_000, 20480, 13_107_200}, + {"200ms", features.ReduceSlotTimeTo200ms, 12_000_000, 30_000_000, 50_000_000, 16384, 10_485_760}, + } { + t.Run(tc.name, func(t *testing.T) { + f := features.NewFeaturesDefault() + if tc.name != "400ms" { + f.EnableFeature(tc.gate, 50) + } + before, err := LimitsForSlot(f, schedule, 99) + require.NoError(t, err) + require.Equal(t, DefaultLimits(), before) + for _, raise := range []bool{false, true} { + if raise { + f.EnableFeature(features.RaiseBlockLimitsTo100m, 0) + } + got, err := LimitsForSlot(f, schedule, 100) + require.NoError(t, err) + account, block := tc.account, tc.block + if raise { + account = account * 100 / 60 + block = block * 100 / 60 + } + require.Equal(t, account, got.WritableAccountCost) + require.Equal(t, block, got.BlockCost) + require.Equal(t, tc.data, got.AllocatedDataSizeDelta) + require.Equal(t, tc.entries-EntryHeaderBytes, got.MaxEntryBytes) + require.LessOrEqual(t, got.MaxEntryBytes, PackEntryBytesMax(tc.shreds, MaxMicroblockBytes)) + require.Equal(t, uint64(DefaultTargetBatchBytes), got.MaxBatchBytes) + } + }) + } +} + +func TestSlotLimitsDoNotLengthenSlotsForLaterGates(t *testing.T) { + f := features.NewFeaturesDefault() + f.EnableFeature(features.ReduceSlotTimeTo200ms, 50) + f.EnableFeature(features.ReduceSlotTimeTo350ms, 150) + schedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100} + for _, slot := range []uint64{100, 199, 200, 400} { + limits, err := LimitsForSlot(f, schedule, slot) + require.NoError(t, err) + require.Equal(t, uint64(30_000_000), limits.BlockCost) + } +} + +func TestSlotLimitsActivationDuringWarmup(t *testing.T) { + f := features.NewFeaturesDefault() + f.EnableFeature(features.ReduceSlotTimeTo200ms, 40) + // Epoch 0: [0,32), epoch 1: [32,96), normal epoch 2: [96,224). + schedule := &sealevel.SysvarEpochSchedule{Warmup: true, SlotsPerEpoch: 128, FirstNormalEpoch: 2, FirstNormalSlot: 96} + before, err := LimitsForSlot(f, schedule, 95) + require.NoError(t, err) + require.Equal(t, uint64(60_000_000), before.BlockCost) + after, err := LimitsForSlot(f, schedule, 96) + require.NoError(t, err) + require.Equal(t, uint64(30_000_000), after.BlockCost) +} + +func TestSlotLimitsRequireScheduleForActiveReductions(t *testing.T) { + f := features.NewFeaturesDefault() + _, err := LimitsForSlot(f, nil, 10) + require.NoError(t, err) + f.EnableFeature(features.ReduceSlotTimeTo200ms, 0) + _, err = LimitsForSlot(f, nil, 10) + require.Error(t, err) + _, err = LimitsForSlot(f, &sealevel.SysvarEpochSchedule{}, 10) + require.Error(t, err) +} diff --git a/pkg/costmodel/transaction_cost.go b/pkg/costmodel/transaction_cost.go index 1dd4f0c71..3a804b037 100644 --- a/pkg/costmodel/transaction_cost.go +++ b/pkg/costmodel/transaction_cost.go @@ -55,6 +55,13 @@ func EstimateTransactionCost(tx *solana.Transaction, feats *features.Features) ( }, nil } + return EstimatePreparedTransactionCost(tx, instrs, limits, feats), nil +} + +// EstimatePreparedTransactionCost reuses successfully parsed instructions and +// compute limits from the same immutable transaction and feature snapshot. +func EstimatePreparedTransactionCost(tx *solana.Transaction, instrs []sealevel.Instruction, limits *sealevel.ComputeBudgetLimits, feats *features.Features) TransactionCost { + writable := writableAccounts(tx) loadedDataCost := loadedAccountsDataSizeCost(limits.LoadedAccountBytes) // Banking-stage admission must reserve at least one page for the fee // payer, including V1 transactions whose inline loaded-data limit is zero. @@ -62,13 +69,13 @@ func EstimateTransactionCost(tx *solana.Transaction, feats *features.Features) ( loadedDataCost = max(loadedDataCost, uint64(HeapCost)) return TransactionCost{ SignatureCost: signatureCost(tx, instrs, feats), - WriteLockCost: writeLockCost(countWriteLocks(tx)), + WriteLockCost: writeLockCost(uint64(len(writable))), DataBytesCost: instructionDataCost(tx), ProgramsExecutionCost: uint64(limits.ComputeUnitLimit), LoadedAccountsDataSizeCost: loadedDataCost, AllocatedAccountsDataSize: estimateAllocDelta(instrs, feats), - WritableAccounts: writableAccounts(tx), - }, nil + WritableAccounts: writable, + } } func signatureCost(tx *solana.Transaction, instrs []sealevel.Instruction, feats *features.Features) uint64 { @@ -177,7 +184,7 @@ func replayInstrsAndAcctMetas(tx *solana.Transaction, feats *features.Features) } upgradeableLoaderPresent := false for _, key := range tx.Message.AccountKeys { - if key.String() == "BPFLoaderUpgradeab1e11111111111111111111111" { + if key == addresses.BpfLoaderUpgradeableAddr { upgradeableLoaderPresent = true break } diff --git a/pkg/merkletree/merkletree.go b/pkg/merkletree/merkletree.go index 988f2633c..0de1d98f3 100644 --- a/pkg/merkletree/merkletree.go +++ b/pkg/merkletree/merkletree.go @@ -46,8 +46,29 @@ func (n *Nodes) GetRoot() (out *[32]byte) { return &n.Nodes[len(n.Nodes)-1] } -// TODO provide a method for memory-efficient Merkle construction when only the root is requested. -// Can be implemented using recursion root level downwards +// HashRoot computes the same root as HashNodes without retaining proof nodes. +// Empty input returns zero. Each level overwrites the preceding level, so only +// one hash per leaf is allocated. Leaves are never modified. +func HashRoot(leaves [][]byte) (root [32]byte) { + if len(leaves) == 0 { + return root + } + if len(leaves) == 1 { + return HashLeaf(leaves[0]) + } + nodes := make([][32]byte, len(leaves)) + for i, leaf := range leaves { + nodes[i] = HashLeaf(leaf) + } + for len(nodes) > 1 { + for i := 0; i < len(nodes); i += 2 { + right := min(i+1, len(nodes)-1) + nodes[i/2] = HashIntermediate(&nodes[i], &nodes[right]) + } + nodes = nodes[:(len(nodes)+1)/2] + } + return nodes[0] +} // HashNodes constructs proof data from a set of leaves. // @@ -97,6 +118,14 @@ func HashNodes(leaves [][]byte) (out Nodes) { // HashLeaf returns the hash of a leaf node. func HashLeaf(data []byte) (out [32]byte) { + if len(data) == 64 { + // Transaction signatures are fixed-width leaves. A single buffer avoids + // incremental hash writes while retaining the leaf domain separator. + var input [65]byte + input[0] = TypeLeaf + copy(input[1:], data) + return sha256.Sum256(input[:]) + } h := sha256.New() h.Write([]byte{TypeLeaf}) h.Write(data) @@ -106,12 +135,11 @@ func HashLeaf(data []byte) (out [32]byte) { // HashIntermediate returns the hash of an intermediate node. func HashIntermediate(left *[32]byte, right *[32]byte) (out [32]byte) { - h := sha256.New() - h.Write([]byte{TypeIntermediate}) - h.Write(left[:]) - h.Write(right[:]) - h.Sum(out[:0]) - return + var input [65]byte + input[0] = TypeIntermediate + copy(input[1:33], left[:]) + copy(input[33:], right[:]) + return sha256.Sum256(input[:]) } // nextLevelLen returns the amount of nodes in the layer above the current one, diff --git a/pkg/merkletree/root_test.go b/pkg/merkletree/root_test.go new file mode 100644 index 000000000..1437c3833 --- /dev/null +++ b/pkg/merkletree/root_test.go @@ -0,0 +1,60 @@ +package merkletree + +import ( + "bytes" + "crypto/sha256" + "fmt" + "testing" +) + +// Independent construction: retain separate levels and use one-shot SHA256 +// over explicit domain-prefixed bytes, without the production hash helpers. +func referenceRoot(leaves [][]byte) [32]byte { + if len(leaves) == 0 { + return [32]byte{} + } + level := make([][32]byte, len(leaves)) + for i, leaf := range leaves { + level[i] = sha256.Sum256(append([]byte{0}, leaf...)) + } + for len(level) > 1 { + next := make([][32]byte, (len(level)+1)/2) + for i := range next { + left, right := level[2*i], level[min(2*i+1, len(level)-1)] + data := append([]byte{1}, left[:]...) + data = append(data, right[:]...) + next[i] = sha256.Sum256(data) + } + level = next + } + return level[0] +} + +func TestHashRootMatchesCanonicalTree(t *testing.T) { + counts := []int{0, 1, 2, 3, 7, 8, 9, 31, 32, 33, 63, 64, 65, 127, 128, 129, 255, 256, 257, 311, 511, 512, 513, 1023, 1024, 1025} + for _, size := range []int{0, 31, 32, 55, 56, 63, 64, 65, 1232} { + for _, count := range counts { + t.Run(fmt.Sprintf("%dx%d", count, size), func(t *testing.T) { + leaves := make([][]byte, count) + for i := range leaves { + leaves[i] = make([]byte, size) + for j := range leaves[i] { + leaves[i][j] = byte(i*71 + i/256 + j*17) + } + } + before := bytes.Join(leaves, nil) + want := referenceRoot(leaves) + if got := HashRoot(leaves); got != want { + t.Fatalf("root %x, want %x", got, want) + } + nodes := HashNodes(leaves) + if got := nodes.GetRoot(); got != nil && *got != want { + t.Fatalf("proof root %x, want %x", *got, want) + } + if !bytes.Equal(before, bytes.Join(leaves, nil)) { + t.Fatal("hashing modified input leaves") + } + }) + } + } +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 10548fdd4..3f87fa287 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -15,8 +15,18 @@ func (t *Timing) AddTiming(d time.Duration) { atomic.AddUint64(&t.SumNanoseconds, uint64(d.Nanoseconds())) } +// StartTiming avoids reading the clock when a caller does not record timings. +func StartTiming(enabled bool) time.Time { + if enabled { + return time.Now() + } + return time.Time{} +} + func (t *Timing) AddTimingSince(start time.Time) { - t.AddTiming(time.Since(start)) + if !start.IsZero() { + t.AddTiming(time.Since(start)) + } } // AccountLoader is the per-slot decomposition of LoadBlockAccounts. Counters @@ -94,15 +104,26 @@ type AccountLoader struct { SysvarCachePublicationEpochRejects uint64 } -// TurbineIngress is the exact per-slot pre-replay pipeline decomposition. +// TurbineIngress records per-slot pre-replay pipeline observations. // It is written to replay_timings.jsonl without high-cardinality metric labels. type TurbineIngress struct { ShredCollection Timing CompletionQueueDelay Timing BlockDecode Timing + // Completion-only parse and outstanding-signature join/verification time. TransactionParse Timing TransactionSigverify Timing ReplayAdmission Timing + // Summed completed prefetched component durations, including any discarded + // optimistic prefix. Overlap reception; not CPU time or additive wall stages. + // Early sigverify includes queueing. + EarlyTransactionParse Timing + EarlyTransactionSigverify Timing + // Completion wait for claimed background parsing/submission, outside BlockDecode. + EarlyPreparationWait Timing + EarlyVerifiedTransactions uint64 + // FullToReady contains the completion stages above, excluding admission. + FullToReady Timing } // VoteRewardDetails decomposes RewardCertificatePreflight and diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 495ed53a8..062802e2d 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -2836,6 +2836,11 @@ func ReplayBlocks( record.TransactionParse.AddTiming(ingressTimings.TransactionParse) record.TransactionSigverify.AddTiming(ingressTimings.TransactionSigverify) record.ReplayAdmission.AddTiming(ingressTimings.ReplayAdmission) + record.EarlyTransactionParse.AddTiming(ingressTimings.EarlyTransactionParse) + record.EarlyTransactionSigverify.AddTiming(ingressTimings.EarlyTransactionSigverify) + record.EarlyPreparationWait.AddTiming(ingressTimings.EarlyPreparationWait) + record.EarlyVerifiedTransactions = ingressTimings.EarlyVerifiedTransactions + record.FullToReady.AddTiming(ingressTimings.FullToReady) } start := time.Now() diff --git a/pkg/replay/chain_state.go b/pkg/replay/chain_state.go index 649146c93..70c404b22 100644 --- a/pkg/replay/chain_state.go +++ b/pkg/replay/chain_state.go @@ -266,3 +266,14 @@ func ChainTipFeatureActive(gate features.FeatureGate) bool { defer chainTipMu.RUnlock() return chainTipFeatures != nil && chainTipFeatures.IsActive(gate) } + +// ChainTipFeatures returns an independent feature snapshot for queued transaction +// preparation. Bank admission checks compatibility again before reuse. +func ChainTipFeatures() *features.Features { + chainTipMu.RLock() + defer chainTipMu.RUnlock() + if chainTipFeatures == nil { + return nil + } + return chainTipFeatures.Clone() +} diff --git a/pkg/replay/transaction_preparation.go b/pkg/replay/transaction_preparation.go new file mode 100644 index 000000000..303dcc3e3 --- /dev/null +++ b/pkg/replay/transaction_preparation.go @@ -0,0 +1,133 @@ +package replay + +import ( + "crypto/sha256" + "encoding/binary" + "sort" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/fees" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" +) + +// TransactionPreparer binds static transaction preparation to an immutable bank +// feature snapshot. Its source must remain immutable for the bank's lifetime. +// Prepared messages must also remain immutable, including their backing bytes. +// Account contents, transaction age and duplicate status are never cached here. +type TransactionPreparer struct { + source *features.Features + feats *features.Features + key [32]byte +} + +// PreparedTransaction contains only message/feature-derived data. Private fields +// prevent callers from substituting an unchecked hash, cost or instruction list. +type PreparedTransaction struct { + tx *solana.Transaction + key [32]byte + hash [32]byte + cost costmodel.TransactionCost + instrs []sealevel.Instruction + instructionAccts [][]sealevel.InstructionAccount + accountMetas []*solana.AccountMeta + limits *sealevel.ComputeBudgetLimits +} + +func NewTransactionPreparer(f *features.Features) *TransactionPreparer { + if f == nil { + return nil + } + clone := f.Clone() + gates := make([]features.FeatureGate, 0, len(*clone)) + for gate := range *clone { + gates = append(gates, gate) + } + sort.Slice(gates, func(i, j int) bool { + if gates[i].Address != gates[j].Address { + return string(gates[i].Address[:]) < string(gates[j].Address[:]) + } + return gates[i].Name < gates[j].Name + }) + h := sha256.New() + var value [8]byte + for _, gate := range gates { + info := (*clone)[gate] + h.Write(gate.Address[:]) + binary.LittleEndian.PutUint64(value[:], uint64(len(gate.Name))) + h.Write(value[:]) + h.Write([]byte(gate.Name)) + if info.Enabled { + h.Write([]byte{1}) + } else { + h.Write([]byte{0}) + } + binary.LittleEndian.PutUint64(value[:], info.ActivationSlot) + h.Write(value[:]) + } + p := &TransactionPreparer{source: f, feats: clone} + copy(p.key[:], h.Sum(nil)) + return p +} + +// Prepare returns nil on a static validation failure. Callers retain their normal +// processing path in that case, preserving its error classification and ordering. +func (p *TransactionPreparer) Prepare(tx *solana.Transaction) *PreparedTransaction { + if p == nil || tx == nil { + return nil + } + if tx.Message.GetVersion() == solana.MessageVersionV1 && !p.feats.IsActive(features.EnableTxV1) { + return nil + } + if txverify.SanitizeTransaction(tx) != nil { + return nil + } + if p.feats.IsActive(features.StaticInstructionLimit) && len(tx.Message.Instructions) > maxInstrTraceCapacity { + return nil + } + instrs, instructionAccts, metas, err := instrsAndAcctMetasFromTx(tx, p.feats) + if err != nil { + return nil + } + limits, err := sealevel.ComputeBudgetLimitsForTransaction(tx, instrs, p.feats) + if err != nil { + return nil + } + hash, err := TransactionMessageHash(tx) + if err != nil { + return nil + } + return &PreparedTransaction{tx: tx, key: p.key, hash: hash, + cost: costmodel.EstimatePreparedTransactionCost(tx, instrs, limits, p.feats), + instrs: instrs, instructionAccts: instructionAccts, accountMetas: metas, limits: limits} +} + +func (p *TransactionPreparer) Matches(prepared *PreparedTransaction, tx *solana.Transaction, f *features.Features) bool { + return p != nil && p.source == f && prepared != nil && prepared.tx == tx && p.key == prepared.key +} + +func (p *PreparedTransaction) MessageHash() [32]byte { return p.hash } + +// Cost returns the estimate; its writable-account slice is read-only. +func (p *PreparedTransaction) Cost() costmodel.TransactionCost { return p.cost } + +// LoadAndExecute reuses preparation only for the same immutable message and a +// matching feature snapshot. All bank-dependent checks still run on every call. +func (p *TransactionPreparer) LoadAndExecute(input LoadAndExecuteTransactionInput, prepared *PreparedTransaction) LoadAndExecuteTransactionOutput { + if input.SlotCtx == nil || p == nil || p.source != input.SlotCtx.Features || !p.Matches(prepared, input.Transaction, input.SlotCtx.Features) { + return LoadAndExecuteTransaction(input) + } + return loadAndExecuteTransaction(input, prepared) +} + +// PayerCanFund keeps strict leader admission, using current payer state while +// sharing the already-validated instructions and compute limits. +func (p *TransactionPreparer) PayerCanFund(slotCtx *sealevel.SlotCtx, tx *solana.Transaction, prepared *PreparedTransaction) error { + if slotCtx == nil || !p.Matches(prepared, tx, slotCtx.Features) { + return fees.PayerCanFund(slotCtx, tx) + } + _, err := fees.ValidateTransactionFeePayer(slotCtx, tx, prepared.instrs, prepared.limits) + return err +} diff --git a/pkg/replay/transaction_processing_pure.go b/pkg/replay/transaction_processing_pure.go index 061703db1..434346460 100644 --- a/pkg/replay/transaction_processing_pure.go +++ b/pkg/replay/transaction_processing_pure.go @@ -3,7 +3,6 @@ package replay import ( "errors" "math" - "time" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/arena" @@ -39,6 +38,10 @@ type LoadAndExecuteTransactionInput struct { // banks also skip writable-account result materialization; RPC simulation // leaves this false to retain the rich result. LeanResult bool + // SkipTimingMetrics omits the detailed transaction and instruction-dispatch + // timings for leader execution. Replay and simulation retain their default + // instrumentation; program-specific instrumentation is independent. + SkipTimingMetrics bool // CapturePreBalances retains pre-fee balances in lean mode. Rich mode // always captures them for RPC compatibility. CapturePreBalances bool @@ -124,6 +127,10 @@ func feeOnlyRollbackAccountsDataSize(slotCtx *sealevel.SlotCtx, tx *solana.Trans } func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExecuteTransactionOutput { + return loadAndExecuteTransaction(input, nil) +} + +func loadAndExecuteTransaction(input LoadAndExecuteTransactionInput, prepared *PreparedTransaction) LoadAndExecuteTransactionOutput { tx := input.Transaction slotCtx := input.SlotCtx @@ -131,64 +138,76 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec input.Arena.Reset() } - if tx == nil || slotCtx == nil || slotCtx.Features == nil { - return sanitizeFailureOutput() - } + var instrs []sealevel.Instruction + var instructionAcctsPerInstr [][]sealevel.InstructionAccount + var txAcctMetas []*solana.AccountMeta + var computeBudgetLimits *sealevel.ComputeBudgetLimits + var err error + start := metrics.StartTiming(false) + if prepared != nil { + instrs, instructionAcctsPerInstr, txAcctMetas = prepared.instrs, prepared.instructionAccts, prepared.accountMetas + computeBudgetLimits = prepared.limits + } else { + if tx == nil || slotCtx == nil || slotCtx.Features == nil { + return sanitizeFailureOutput() + } - // Match Agave's Bank verification order: once a transaction has decoded as - // v1, the feature gate is checked before structural sanitization. - if tx.Message.GetVersion() == solana.MessageVersionV1 && !slotCtx.Features.IsActive(features.EnableTxV1) { - return LoadAndExecuteTransactionOutput{ - ProcessingResult: TransactionProcessingResult{ - TransactionError: &TransactionError{ - ErrorType: TransactionErrorUnsupportedVersion, - InstructionError: TxErrUnsupportedVersion, + // Match Agave's Bank verification order: once a transaction has decoded as + // v1, the feature gate is checked before structural sanitization. + if tx.Message.GetVersion() == solana.MessageVersionV1 && !slotCtx.Features.IsActive(features.EnableTxV1) { + return LoadAndExecuteTransactionOutput{ + ProcessingResult: TransactionProcessingResult{ + TransactionError: &TransactionError{ + ErrorType: TransactionErrorUnsupportedVersion, + InstructionError: TxErrUnsupportedVersion, + }, }, - }, + } + } + // Reject malformed transactions before account-indexed code can observe + // them. The helper also handles Mithril's already-resolved v0 messages. + if err := txverify.SanitizeTransaction(tx); err != nil { + return sanitizeFailureOutput() + } + // Mirror block-replay's StaticInstructionLimit cap so pre-activation + // clusters fail mid-execution like Agave instead of SanitizeFailure. + if slotCtx.Features.IsActive(features.StaticInstructionLimit) && + len(tx.Message.Instructions) > maxInstrTraceCapacity { + return sanitizeFailureOutput() } - } - // Reject malformed transactions before account-indexed code can observe - // them. The helper also handles Mithril's already-resolved v0 messages. - if err := txverify.SanitizeTransaction(tx); err != nil { - return sanitizeFailureOutput() - } - // Mirror block-replay's StaticInstructionLimit cap so pre-activation - // clusters fail mid-execution like Agave instead of SanitizeFailure. - if slotCtx.Features.IsActive(features.StaticInstructionLimit) && - len(tx.Message.Instructions) > maxInstrTraceCapacity { - return sanitizeFailureOutput() - } - // Parse instructions and account metas - start := time.Now() - instrs, instructionAcctsPerInstr, txAcctMetas, err := instrsAndAcctMetasFromTx(tx, slotCtx.Features) - if err != nil { - return LoadAndExecuteTransactionOutput{ - ProcessingResult: TransactionProcessingResult{ - TransactionError: &TransactionError{ - ErrorType: TransactionErrorSanitizeFailure, - InstructionError: err, + // Parse instructions and account metas + start = metrics.StartTiming(!input.SkipTimingMetrics) + instrs, instructionAcctsPerInstr, txAcctMetas, err = instrsAndAcctMetasFromTx(tx, slotCtx.Features) + if err != nil { + return LoadAndExecuteTransactionOutput{ + ProcessingResult: TransactionProcessingResult{ + TransactionError: &TransactionError{ + ErrorType: TransactionErrorSanitizeFailure, + InstructionError: err, + }, }, - }, + } } - } - metrics.GlobalBlockReplay.InstructionsAndAccountMetasFromTx.AddTimingSince(start) - - // Compute budget limits - start = time.Now() - computeBudgetLimits, err := sealevel.ComputeBudgetLimitsForTransaction(tx, instrs, slotCtx.Features) - if err != nil { - return LoadAndExecuteTransactionOutput{ - ProcessingResult: TransactionProcessingResult{ - TransactionError: &TransactionError{ - ErrorType: TransactionErrorInstructionError, - InstructionError: err, + metrics.GlobalBlockReplay.InstructionsAndAccountMetasFromTx.AddTimingSince(start) + + // Compute budget limits + start = metrics.StartTiming(!input.SkipTimingMetrics) + computeBudgetLimits, err = sealevel.ComputeBudgetLimitsForTransaction(tx, instrs, slotCtx.Features) + if err != nil { + return LoadAndExecuteTransactionOutput{ + ProcessingResult: TransactionProcessingResult{ + TransactionError: &TransactionError{ + ErrorType: TransactionErrorInstructionError, + InstructionError: err, + }, }, - }, - Instrs: instrs, + Instrs: instrs, + } } + metrics.GlobalBlockReplay.ComputeBudgetExecutionInstructions.AddTimingSince(start) + } - metrics.GlobalBlockReplay.ComputeBudgetExecutionInstructions.AddTimingSince(start) // Validate transaction age if !sealevel.IsTransactionAgeValid(tx, instrs, slotCtx) { @@ -236,7 +255,7 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec } // Load and validate accounts - start = time.Now() + start = metrics.StartTiming(!input.SkipTimingMetrics) instructionsSysvarIdx := instructionsSysvarAccountIndex(tx) var instrsAcct *accounts.Account if instructionsSysvarIdx >= 0 { @@ -303,6 +322,7 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec execCtx.TransactionContext.Signature = tx.Signatures[0] execCtx.TransactionContext.BorrowedAccountArena = input.Arena execCtx.IsSimulation = input.IsSimulation + execCtx.SkipTimingMetrics = input.SkipTimingMetrics execCtx.RecordInnerInstructions = input.RecordInnerInstructions // Capture pre-balance lamports (before fee deduction) @@ -332,7 +352,7 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec // Calculate and deduct fees. RentForSlot supplies the exemption // minimum so a rent-exempt payer is rejected before instructions run. - start = time.Now() + start = metrics.StartTiming(!input.SkipTimingMetrics) txFeeInfo, _, err := fees.CalculateAndDeductTxFees(tx, input.TxMeta, instrs, &execCtx.TransactionContext.Accounts, computeBudgetLimits, slotCtx.Features, fees.RentForSlot(slotCtx), input.IsSimulation) if err != nil { errType, accountIndex := feePayerTransactionError(err) @@ -355,7 +375,7 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec metrics.GlobalBlockReplay.CalcAndDeductFees.AddTimingSince(start) // Read rent sysvar - start = time.Now() + start = metrics.StartTiming(!input.SkipTimingMetrics) rentSysvar, err := sealevel.ReadRentSysvar(execCtx) if err != nil { // Rent sysvar unreadable; return cleanly so the RPC worker @@ -374,18 +394,18 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec metrics.GlobalBlockReplay.ReadRentSysvar.AddTimingSince(start) // Set rent-exempt rent epoch max and compute pre-tx rent states - start = time.Now() + start = metrics.StartTiming(!input.SkipTimingMetrics) rent.MaybeSetRentExemptRentEpochMax(slotCtx, &rentSysvar, &execCtx.Features, &execCtx.TransactionContext.Accounts) preTxRentStates := rent.NewRentStateInfo(&rentSysvar, execCtx.TransactionContext, &execCtx.Features) metrics.GlobalBlockReplay.PreTxRentStates.AddTimingSince(start) // Execute all instructions var instrErr error - start = time.Now() + start = metrics.StartTiming(!input.SkipTimingMetrics) for instrIdx, instr := range tx.Message.Instructions { execCtx.SetCurrentTopLevelInstr(uint8(instrIdx)) if instructionsSysvarIdx >= 0 { - ixStart := time.Now() + ixStart := metrics.StartTiming(!input.SkipTimingMetrics) err = fixupInstructionsSysvarAcct(execCtx, instructionsSysvarIdx, uint16(instrIdx)) if err != nil { instrErr = err @@ -421,7 +441,7 @@ func LoadAndExecuteTransaction(input LoadAndExecuteTransactionInput) LoadAndExec metrics.GlobalBlockReplay.IxLoop.AddTimingSince(start) // Check rent state transitions - start = time.Now() + start = metrics.StartTiming(!input.SkipTimingMetrics) postTxRentStates := rent.NewRentStateInfo(&rentSysvar, execCtx.TransactionContext, &execCtx.Features) rentStateErr := rent.VerifyRentStateChanges(preTxRentStates, postTxRentStates, execCtx.TransactionContext) metrics.GlobalBlockReplay.PostTxRentStates.AddTimingSince(start) diff --git a/pkg/sealevel/execution_ctx.go b/pkg/sealevel/execution_ctx.go index 348b4647c..ba54cc267 100644 --- a/pkg/sealevel/execution_ctx.go +++ b/pkg/sealevel/execution_ctx.go @@ -5,7 +5,6 @@ import ( "fmt" "sync" "sync/atomic" - "time" "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/accountsdb" @@ -19,6 +18,9 @@ import ( ) type ExecutionCtx struct { + // SkipTimingMetrics disables instruction-dispatch timing collection for + // leader execution; it never changes instruction validation or CU charging. + SkipTimingMetrics bool Log Logger Accounts accounts.Accounts TransactionContext *TransactionCtx @@ -237,14 +239,14 @@ func (execCtx *ExecutionCtx) PrepareInstruction(ix Instruction, signers []solana } func (execCtx *ExecutionCtx) ProcessInstruction(instrData []byte, instructionAccts []InstructionAccount, programIndices []uint64) error { - start := time.Now() + start := metrics.StartTiming(!execCtx.SkipTimingMetrics) nextInstrCtx, err := execCtx.TransactionContext.NextInstructionCtx() if err != nil { return err } metrics.GlobalBlockReplay.GetNextIxCtx.AddTimingSince(start) - start = time.Now() + start = metrics.StartTiming(!execCtx.SkipTimingMetrics) nextInstrCtx.Configure(programIndices, instructionAccts, instrData) metrics.GlobalBlockReplay.NextIxCtxConfigure.AddTimingSince(start) @@ -267,7 +269,7 @@ func (execCtx *ExecutionCtx) ProcessInstruction(instrData []byte, instructionAcc }) } - start = time.Now() + start = metrics.StartTiming(!execCtx.SkipTimingMetrics) err = execCtx.Push() if err != nil { return err @@ -283,7 +285,7 @@ func (execCtx *ExecutionCtx) ProcessInstruction(instrData []byte, instructionAcc err1 := execCtx.ExecuteInstruction() - start = time.Now() + start = metrics.StartTiming(!execCtx.SkipTimingMetrics) err2 := execCtx.Pop() metrics.GlobalBlockReplay.IxPop.AddTimingSince(start) @@ -304,7 +306,7 @@ func (execCtx *ExecutionCtx) AddModifiedVoteState(pubkey solana.PublicKey, state } func (execCtx *ExecutionCtx) ExecuteInstruction() error { - start := time.Now() + start := metrics.StartTiming(!execCtx.SkipTimingMetrics) txCtx := execCtx.TransactionContext instrCtx, err := txCtx.CurrentInstructionCtx() @@ -334,7 +336,7 @@ func (execCtx *ExecutionCtx) ExecuteInstruction() error { } metrics.GlobalBlockReplay.ExecIxResolveNativeProgram.AddTimingSince(start) - start = time.Now() + start = metrics.StartTiming(!execCtx.SkipTimingMetrics) err = nativeProgramFn(execCtx) switch nativeProgramStr { case a.SystemProgramAddrStr: diff --git a/pkg/sigverify/config_policy_test.go b/pkg/sigverify/config_policy_test.go new file mode 100644 index 000000000..52e8948ce --- /dev/null +++ b/pkg/sigverify/config_policy_test.go @@ -0,0 +1,63 @@ +package sigverify + +import ( + "runtime" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestResolveConfigPolicyDefaultsAndOverrides(t *testing.T) { + zero, err := ResolveConfig(Config{}) + require.NoError(t, err) + require.Equal(t, BackendAuto, zero.Backend) + require.Equal(t, min(2, runtime.GOMAXPROCS(0)), zero.Workers) + require.Equal(t, 8, zero.BatchTarget) + require.False(t, zero.DisableShredOverlap) + defaults, err := ResolveConfig(Defaults()) + require.NoError(t, err) + require.Equal(t, zero, defaults) + + override := Config{Backend: BackendGeneric, Workers: 4, BatchTarget: 4, DisableShredOverlap: true} + resolved, err := ResolveConfig(override) + require.NoError(t, err) + require.Equal(t, override, resolved) +} + +func TestInvalidPolicyDoesNotLatchBackend(t *testing.T) { + if !inChild() { + out, err := runConfigureChild(t, t.Name()) + require.NoError(t, err, "child output:\n%s", out) + require.Contains(t, out, "PASS") + return + } + before := Cfg + for _, cfg := range []Config{ + {Backend: BackendGeneric, Workers: -1}, + {Backend: BackendGeneric, BatchTarget: -1}, + {Backend: BackendGeneric, BatchTarget: 3}, + {Backend: BackendGeneric, BatchTarget: 16}, + } { + _, err := Configure(cfg) + require.Error(t, err) + require.Equal(t, before, Cfg, "invalid policy must not become live") + require.Empty(t, configuredBackend, "invalid policy must not latch the backend") + } + _, err := Configure(Config{Backend: BackendGeneric, Workers: 2, BatchTarget: 4}) + require.NoError(t, err, "valid configuration must remain possible after rejection") + require.Equal(t, 2, TransactionWorkers()) + require.Equal(t, 4, TransactionBatchTarget()) +} + +func TestTransactionPolicyWithoutConfigure(t *testing.T) { + if !inChild() { + out, err := runConfigureChild(t, t.Name()) + require.NoError(t, err, "child output:\n%s", out) + require.Contains(t, out, "PASS") + return + } + Cfg = Config{} + require.Equal(t, min(2, runtime.GOMAXPROCS(0)), TransactionWorkers()) + require.Equal(t, 8, TransactionBatchTarget()) + require.False(t, Cfg.DisableShredOverlap) +} diff --git a/pkg/sigverify/sigverify.go b/pkg/sigverify/sigverify.go index fae00447d..a0502712c 100644 --- a/pkg/sigverify/sigverify.go +++ b/pkg/sigverify/sigverify.go @@ -20,6 +20,7 @@ package sigverify import ( "fmt" + "runtime" "sync" narya "github.com/Overclock-Validator/narya-ed25519/ed25519" @@ -40,15 +41,60 @@ const ( BackendStdlib = "stdlib" ) -// Config selects the verification backend. It is deliberately tiny: the -// library's own defaults are good, and every knob here is a consensus-visible -// or performance-visible choice that an operator should have to state. +// Config selects the backend and the Turbine transaction-verification policy. +// Worker and batching settings do not change the TPU or fallback replay pools. type Config struct { - Backend string + Backend string + Workers int + BatchTarget int + DisableShredOverlap bool } // Defaults returns the configuration used when the operator sets nothing. -func Defaults() Config { return Config{Backend: BackendAuto} } +func Defaults() Config { return Config{Backend: BackendAuto, BatchTarget: BatchTarget} } + +// ResolveConfig validates every setting before the one-shot backend selection. +// Zero workers selects at most two transaction-verification workers; zero batch +// target uses eight signatures. Available short batches are never held to fill. +func ResolveConfig(cfg Config) (Config, error) { + if cfg.Backend == "" { + cfg.Backend = BackendAuto + } + switch cfg.Backend { + case BackendAuto, BackendR51, BackendGeneric, BackendStdlib: + default: + return Config{}, fmt.Errorf("sigverify.backend must be one of %q, %q, %q, %q; got %q", BackendAuto, BackendR51, BackendGeneric, BackendStdlib, cfg.Backend) + } + if cfg.Workers < 0 { + return Config{}, fmt.Errorf("sigverify.workers must be >= 0; got %d", cfg.Workers) + } + if cfg.Workers == 0 { + cfg.Workers = min(2, max(1, runtime.GOMAXPROCS(0))) + } + if cfg.BatchTarget == 0 { + cfg.BatchTarget = BatchTarget + } + if cfg.BatchTarget != 4 && cfg.BatchTarget != 8 { + return Config{}, fmt.Errorf("sigverify.batch_target must be 4 or 8 (0 uses 8); got %d", cfg.BatchTarget) + } + return cfg, nil +} + +// TransactionWorkers also supports callers that do not run node Configure. +func TransactionWorkers() int { + if Cfg.Workers > 0 { + return Cfg.Workers + } + return min(2, max(1, runtime.GOMAXPROCS(0))) +} + +// TransactionBatchTarget also supports the zero configuration outside startup. +func TransactionBatchTarget() int { + if Cfg.BatchTarget == 4 { + return 4 + } + return BatchTarget +} // Cfg is the live configuration, set once by Configure during startup and // read-only afterwards. It follows the same shape as replay.TrailingVerifierCfg. @@ -64,10 +110,6 @@ var Cfg = Defaults() // underlying library pins its backend on first use and a late switch would // leave the process in a state neither caller asked for. func Configure(cfg Config) (string, error) { - if cfg.Backend == "" { - cfg.Backend = Defaults().Backend - } - configureMu.Lock() defer configureMu.Unlock() @@ -79,12 +121,10 @@ func Configure(cfg Config) (string, error) { // Validate before publishing anything. Assigning Cfg first would leave a // rejected backend name visible to Backend() and to the startup log. - switch cfg.Backend { - case BackendAuto, BackendR51, BackendGeneric, BackendStdlib: - default: - return "", fmt.Errorf( - "sigverify.backend must be one of %q, %q, %q, %q; got %q", - BackendAuto, BackendR51, BackendGeneric, BackendStdlib, cfg.Backend) + var err error + cfg, err = ResolveConfig(cfg) + if err != nil { + return "", err } resolved, err := installBackend(cfg.Backend) diff --git a/pkg/statsd/statsd.go b/pkg/statsd/statsd.go index bbf3eea9c..2418cc025 100644 --- a/pkg/statsd/statsd.go +++ b/pkg/statsd/statsd.go @@ -111,6 +111,7 @@ var ( TxsPerBlock = Metric{"txs_per_block"} SnapshotTarBytesRead = Metric{"snapshot_tar_bytes_read"} SlotReplays = Metric{"slot_replays"} + BlockProductionEntrySerializationErrors = Metric{"block_production_entry_serialization_errors_total"} BlockProductionLeaderSlots = Metric{"block_production_leader_slots_total"} BlockProductionLeaderSlotTerminals = Metric{"block_production_leader_slot_terminals_total"} BlockProductionParentReady = Metric{"block_production_parent_ready_activations_total"} @@ -124,6 +125,13 @@ var ( TurbineBlockDecode = Metric{"turbine_block_decode_duration_seconds"} TurbineTransactionParse = Metric{"turbine_transaction_parse_duration_seconds"} TurbineTransactionSigverify = Metric{"turbine_transaction_sigverify_duration_seconds"} + // Early durations sum elapsed component work, including verifier queueing; + // they overlap shred collection and are neither CPU nor pipeline wall time. + TurbineEarlyTransactionParse = Metric{"turbine_early_transaction_parse_duration_seconds"} + TurbineEarlyTransactionSigverify = Metric{"turbine_early_transaction_sigverify_elapsed_seconds"} + TurbineEarlyPreparationWait = Metric{"turbine_early_preparation_wait_seconds"} + TurbineEarlyVerifiedTransactions = Metric{"turbine_early_verified_transactions_total"} + TurbineFullToReady = Metric{"turbine_full_to_ready_duration_seconds"} // ReplaySigverifyGroup times one drained group of transaction signatures // and ReplaySigverifyGroupSignatures counts how many signatures were in it. // The pair is what tells an operator whether batching is actually happening: @@ -229,11 +237,12 @@ var MetricToType = map[Metric]metricType{ SlotReplayDurationMs: TimingT, TxsPerBlock: TimingT, - SnapshotTarBytesRead: CountT, - SlotReplays: CountT, - BlockProductionLeaderSlots: CountT, - BlockProductionLeaderSlotTerminals: CountT, - BlockProductionParentReady: CountT, + SnapshotTarBytesRead: CountT, + SlotReplays: CountT, + BlockProductionEntrySerializationErrors: CountT, + BlockProductionLeaderSlots: CountT, + BlockProductionLeaderSlotTerminals: CountT, + BlockProductionParentReady: CountT, BlockProductionParentReadyAge: TimingT, BlockProductionStartCutoffLate: TimingT, @@ -245,6 +254,11 @@ var MetricToType = map[Metric]metricType{ TurbineBlockDecode: TimingT, TurbineTransactionParse: TimingT, TurbineTransactionSigverify: TimingT, + TurbineEarlyTransactionParse: TimingT, + TurbineEarlyTransactionSigverify: TimingT, + TurbineEarlyPreparationWait: TimingT, + TurbineEarlyVerifiedTransactions: CountT, + TurbineFullToReady: TimingT, ReplaySigverifyGroup: TimingT, ReplaySigverifyGroupSignatures: CountT, TurbineReplayAdmission: TimingT, @@ -335,13 +349,14 @@ var MetricToLabels = map[Metric][]string{ TasksIndexEntryBuilderLatency: {}, TasksAppendVecCopyingLatency: {}, - SlotReplayDurationMs: {}, - TxsPerBlock: {}, - SnapshotTarBytesRead: {}, - SlotReplays: {}, - BlockProductionLeaderSlots: {"outcome", "reason"}, - BlockProductionLeaderSlotTerminals: {"outcome", "terminal", "cause"}, - BlockProductionParentReady: {"activation", "status"}, + SlotReplayDurationMs: {}, + TxsPerBlock: {}, + SnapshotTarBytesRead: {}, + SlotReplays: {}, + BlockProductionEntrySerializationErrors: {}, + BlockProductionLeaderSlots: {"outcome", "reason"}, + BlockProductionLeaderSlotTerminals: {"outcome", "terminal", "cause"}, + BlockProductionParentReady: {"activation", "status"}, BlockProductionParentReadyAge: {"activation"}, BlockProductionStartCutoffLate: {"phase"}, @@ -353,6 +368,11 @@ var MetricToLabels = map[Metric][]string{ TurbineBlockDecode: {}, TurbineTransactionParse: {}, TurbineTransactionSigverify: {}, + TurbineEarlyTransactionParse: {}, + TurbineEarlyTransactionSigverify: {}, + TurbineEarlyPreparationWait: {}, + TurbineEarlyVerifiedTransactions: {}, + TurbineFullToReady: {}, ReplaySigverifyGroup: {}, ReplaySigverifyGroupSignatures: {}, TurbineReplayAdmission: {}, @@ -390,6 +410,10 @@ var MetricToBuckets = map[Metric][]float64{ TurbineBlockDecode: turbinePipelineDurationBuckets, TurbineTransactionParse: turbinePipelineDurationBuckets, TurbineTransactionSigverify: turbinePipelineDurationBuckets, + TurbineEarlyTransactionParse: turbinePipelineDurationBuckets, + TurbineEarlyTransactionSigverify: turbinePipelineDurationBuckets, + TurbineEarlyPreparationWait: turbinePipelineDurationBuckets, + TurbineFullToReady: turbinePipelineDurationBuckets, ReplaySigverifyGroup: turbinePipelineDurationBuckets, TurbineReplayAdmission: turbinePipelineDurationBuckets, AlpenglowVoteRewards: turbinePipelineDurationBuckets, diff --git a/pkg/statsd/statsd_test.go b/pkg/statsd/statsd_test.go index 830ea99eb..6ec3cd0c5 100644 --- a/pkg/statsd/statsd_test.go +++ b/pkg/statsd/statsd_test.go @@ -203,6 +203,10 @@ func TestTurbinePipelineDurationMetricsUseSecondsAndBoundedSchema(t *testing.T) TurbineBlockDecode, TurbineTransactionParse, TurbineTransactionSigverify, + TurbineEarlyTransactionParse, + TurbineEarlyTransactionSigverify, + TurbineEarlyPreparationWait, + TurbineFullToReady, TurbineReplayAdmission, } duration := 25 * time.Millisecond @@ -228,6 +232,11 @@ func TestTurbinePipelineDurationMetricsUseSecondsAndBoundedSchema(t *testing.T) } } +func TestTurbineEarlyVerifiedTransactionsHasBoundedCountSchema(t *testing.T) { + assert.Equal(t, CountT, MetricToType[TurbineEarlyVerifiedTransactions]) + assert.Equal(t, []string{}, MetricToLabels[TurbineEarlyVerifiedTransactions]) +} + func TestBlockProductionMetricLabelsStayBounded(t *testing.T) { assert.Equal(t, []string{"outcome", "reason"}, MetricToLabels[BlockProductionLeaderSlots]) assert.Equal(t, []string{"outcome", "terminal", "cause"}, MetricToLabels[BlockProductionLeaderSlotTerminals]) diff --git a/pkg/tpu/txfixture/readonly_pair.go b/pkg/tpu/txfixture/readonly_pair.go new file mode 100644 index 000000000..c1de0a074 --- /dev/null +++ b/pkg/tpu/txfixture/readonly_pair.go @@ -0,0 +1,42 @@ +package txfixture + +import ( + "crypto/ed25519" + "fmt" + + "github.com/gagliardetto/solana-go" +) + +const ReadonlyPairPoolSize = 128 +const ReadonlyPairCapacity = ReadonlyPairPoolSize * (ReadonlyPairPoolSize - 1) + +// ReadonlyPairWire builds the 198-byte, single-signature, zero-instruction +// workload used for leader packing tests. Varying ordered pairs of existing +// readonly accounts gives distinct messages without adding instructions or +// forcing a lookup of a new nonexistent account for every transaction. +// Reusing an ordinal requires a different payer or recent blockhash. +func ReadonlyPairWire(key ed25519.PrivateKey, hash solana.Hash, pool []solana.PublicKey, ordinal int) ([]byte, error) { + if len(key) != ed25519.PrivateKeySize || len(pool) != ReadonlyPairPoolSize || ordinal < 0 || ordinal >= ReadonlyPairCapacity { + return nil, fmt.Errorf("invalid readonly-pair fixture key, pool or ordinal") + } + n := ordinal * 7919 % ReadonlyPairCapacity + a, b := n/(ReadonlyPairPoolSize-1), n%(ReadonlyPairPoolSize-1) + if b >= a { + b++ + } + payer := solana.PublicKeyFromBytes(key.Public().(ed25519.PublicKey)) + if payer == pool[a] || payer == pool[b] || pool[a] == pool[b] { + return nil, fmt.Errorf("readonly-pair accounts must be distinct") + } + message := make([]byte, 0, 133) + message = append(message, 1, 0, 2, 3) + message = append(message, payer[:]...) + message = append(message, pool[a][:]...) + message = append(message, pool[b][:]...) + message = append(message, hash[:]...) + message = append(message, 0) + wire := make([]byte, 0, 198) + wire = append(wire, 1) + wire = append(wire, ed25519.Sign(key, message)...) + return append(wire, message...), nil +} diff --git a/pkg/tpu/txfixture/readonly_pair_test.go b/pkg/tpu/txfixture/readonly_pair_test.go new file mode 100644 index 000000000..70efa16dd --- /dev/null +++ b/pkg/tpu/txfixture/readonly_pair_test.go @@ -0,0 +1,68 @@ +package txfixture + +import ( + "crypto/ed25519" + "crypto/sha256" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Reproduce the full preloaded four-slot experiment without RPC, funds or sends. +func TestReadonlyPair200KDistinctMessages(t *testing.T) { + keys := make([]ed25519.PrivateKey, 8) + for i := range keys { + seed := sha256.Sum256([]byte{byte(i), 73}) + keys[i] = ed25519.NewKeyFromSeed(seed[:]) + } + pool := make([]solana.PublicKey, ReadonlyPairPoolSize) + for i := range pool { + pool[i] = solana.PublicKey{byte(i + 1), 77} + } + seen := make(map[[32]byte]struct{}, 200000) + for i := 0; i < 200000; i++ { + hash, index := solana.Hash{1}, i + if i >= 120000 { + hash, index = solana.Hash{2}, i-120000 + } + wire, err := ReadonlyPairWire(keys[index%8], hash, pool, index/8) + require.NoError(t, err) + require.Len(t, wire, 198) + h := sha256.Sum256(wire[65:]) + if _, ok := seen[h]; ok { + t.Fatalf("duplicate message %d", i) + } + seen[h] = struct{}{} + if i%ReadonlyPairCapacity == 0 || i == 119999 || i == 120000 || i == 199999 { + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + require.Len(t, tx.Signatures, 1) + require.Empty(t, tx.Message.Instructions) + require.Equal(t, hash, tx.Message.RecentBlockhash) + require.True(t, ed25519.Verify(ed25519.PublicKey(tx.Message.AccountKeys[0][:]), wire[65:], wire[1:65])) + } + } +} + +func TestReadonlyPairRejectsInvalidInputs(t *testing.T) { + key := ed25519.PrivateKey(PayerPrivateKey()) + pool := make([]solana.PublicKey, ReadonlyPairPoolSize) + for i := range pool { + pool[i] = solana.PublicKey{byte(i + 1), 77} + } + for _, ordinal := range []int{-1, ReadonlyPairCapacity} { + _, err := ReadonlyPairWire(key, TestBlockhash(), pool, ordinal) + require.Error(t, err) + } + _, err := ReadonlyPairWire(nil, TestBlockhash(), pool, 0) + require.Error(t, err) + _, err = ReadonlyPairWire(key, TestBlockhash(), nil, 0) + require.Error(t, err) + pool[0] = solana.PublicKeyFromBytes(key.Public().(ed25519.PublicKey)) + _, err = ReadonlyPairWire(key, TestBlockhash(), pool, 0) + require.Error(t, err) + pool[0], pool[1] = solana.PublicKey{9}, solana.PublicKey{9} + _, err = ReadonlyPairWire(key, TestBlockhash(), pool, 0) + require.Error(t, err) +} diff --git a/pkg/turbine/assembler.go b/pkg/turbine/assembler.go index 9b69f4370..0dcff2fa6 100644 --- a/pkg/turbine/assembler.go +++ b/pkg/turbine/assembler.go @@ -10,6 +10,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/block" "github.com/Overclock-Validator/mithril/pkg/statsd" + "github.com/Overclock-Validator/mithril/pkg/turbine/internal/rsrecover" "github.com/gagliardetto/solana-go" "github.com/klauspost/reedsolomon" ) @@ -43,20 +44,26 @@ const ( ) type SlotAssembler struct { - mu sync.Mutex - slots map[uint64]*slotState - completedSlots map[uint64]struct{} - knownBlockIDs map[uint64]solana.Hash - rejectedBlockIDs map[uint64]map[solana.Hash]struct{} - protectedKnownIDs map[uint64]struct{} - protectedBlockIDs map[uint64]struct{} - priorityRepairSlots map[uint64]struct{} - priorityRepairOrder []uint64 - encoders map[fecLayout]reedsolomon.Encoder - partialShredObs map[uint64]PartialShredObservation // shreds seen for slots that never became full (retained for skip observability) - retentionFloor uint64 // when non-zero, slots >= floor are never "too old" (repair catchup holds a window far behind the live edge) - edgeScanLag uint64 // how far behind the shred edge the freshness-repair scan reaches (0 = repairScanSlotWindow) - maxObservedSlot uint64 + mu sync.Mutex + slots map[uint64]*slotState + completedSlots map[uint64]struct{} + knownBlockIDs map[uint64]solana.Hash + rejectedBlockIDs map[uint64]map[solana.Hash]struct{} + protectedKnownIDs map[uint64]struct{} + protectedBlockIDs map[uint64]struct{} + priorityRepairSlots map[uint64]struct{} + priorityRepairOrder []uint64 + encoders map[fecLayout]reedsolomon.Encoder + partialShredObs map[uint64]PartialShredObservation // shreds seen for slots that never became full (retained for skip observability) + retentionFloor uint64 // when non-zero, slots >= floor are never "too old" (repair catchup holds a window far behind the live edge) + edgeScanLag uint64 // how far behind the shred edge the freshness-repair scan reaches (0 = repairScanSlotWindow) + maxObservedSlot uint64 + // Age sweeps depend on the edge, repair floor, and mutations that can add + // old metadata or release a completing generation's protected identities. + retentionSwept bool + retentionDirty bool + retentionSweepEdge uint64 + retentionSweepFloor uint64 highestFullSlot uint64 // monotonic: highest slot reconstructed from shreds ("full", Agave SlotMeta/is_full sense) recoveredDataShreds uint64 usefulRepairShreds uint64 // distinct data shreds delivered BY repair (the throughput signal) @@ -72,6 +79,7 @@ type SlotAssembler struct { // Configured before ingestion; tests may replace it with a blocking probe. // Production uses the process-wide bounded transaction verifier. verifyTransactions func(context.Context, *block.Block) error + entryPrefetch *entryPrefetchPool } type SlotRepairRequest struct { @@ -91,14 +99,15 @@ type PartialShredObservation struct { } type slotState struct { - slot uint64 - parentSlot uint64 - shreds map[uint32]*Shred - fecSets map[uint32]*fecState - lastIndex uint32 - haveLast bool - shredVer uint16 - firstParent bool + pipelineTrace *entryPipelineTrace + slot uint64 + parentSlot uint64 + shreds map[uint32]*Shred + fecSets map[uint32]*fecState + lastIndex uint32 + haveLast bool + shredVer uint16 + firstParent bool // Observability: when the slot's first shred was accepted, and how many of // its shreds arrived via repair rather than turbine. @@ -106,7 +115,10 @@ type slotState struct { fullAt time.Time repairedShreds int // completing makes the immutable full state a single-owner generation token. - completing bool + completing bool + batchIndex *entryBatchIndex + completeBatches []shredBatchRange + prefetch *slotEntryPrefetch // Assembly failures for this slot (mixed variants/signatures, FEC layout // conflicts, ...). A slot frozen below completion while repair responses // flow is usually poisoned state — the latest error names the poison. @@ -161,6 +173,7 @@ type fecState struct { haveSig bool dataVariant byte codeVariant byte + rootCache *authenticatedFECRoot } func NewSlotAssembler() *SlotAssembler { @@ -174,7 +187,6 @@ func NewSlotAssembler() *SlotAssembler { priorityRepairSlots: make(map[uint64]struct{}), partialShredObs: make(map[uint64]PartialShredObservation), encoders: make(map[fecLayout]reedsolomon.Encoder), - verifyTransactions: validateBlockTransactionsContext, } } @@ -185,6 +197,7 @@ func (a *SlotAssembler) recordPartialObsLocked(state *slotState) { if state == nil || len(state.shreds) == 0 { return } + a.retentionDirty = true a.partialShredObs[state.slot] = PartialShredObservation{ DataShreds: len(state.shreds), RepairedShreds: state.repairedShreds, @@ -239,6 +252,13 @@ func (a *SlotAssembler) AddShredFrom(shred *Shred, fromRepair bool) (*block.Bloc // reconstructable it returns a single immutable completion token; decoding, // parsing, and signature verification must happen after this method unlocks. func (a *SlotAssembler) addShredFrom(shred *Shred, fromRepair bool) (*slotCompletionWork, error) { + return a.addShredFromWithRoot(shred, fromRepair, nil) +} + +// addShredFromWithRoot accepts an optional result from successful authentication +// of this immutable shred. Unauthenticated callers and spool hydration use nil +// and retain the normal root-computation fallback. +func (a *SlotAssembler) addShredFromWithRoot(shred *Shred, fromRepair bool, root *solana.Hash) (*slotCompletionWork, error) { if shred == nil { return nil, nil } @@ -287,9 +307,16 @@ func (a *SlotAssembler) addShredFrom(shred *Shred, fromRepair bool) (*slotComple state.noteError(err) return nil, err } + state.traceAcceptedShred(shred) + a.notePrefetchShredLocked(state, shred) if state.firstShredAt.IsZero() { state.firstShredAt = time.Now() } + if root != nil { + if fec := state.fecSets[shred.FECSetIndex]; fec != nil { + fec.rememberAuthenticatedRoot(shred, *root) + } + } recovered, err := a.recoverFEC(state, shred.FECSetIndex) if err != nil { @@ -303,10 +330,13 @@ func (a *SlotAssembler) addShredFrom(shred *Shred, fromRepair bool) (*slotComple return nil, err } if err == nil { + state.traceAcceptedShred(recoveredShred) + a.notePrefetchShredLocked(state, recoveredShred) a.recoveredDataShreds++ } } + a.prefetchEntriesLocked(state) if !state.complete() { return nil, nil } @@ -318,6 +348,9 @@ func (a *SlotAssembler) claimCompletionLocked(state *slotState, reportNonCanonic return nil } now := time.Now() + if state.pipelineTrace != nil { + state.pipelineTrace.sealed = true + } observeCollection := state.fullAt.IsZero() if observeCollection { state.fullAt = now @@ -348,6 +381,7 @@ func (a *SlotAssembler) abortCompletion(work *slotCompletionWork) { } a.mu.Lock() if a.slots[work.state.slot] == work.state && work.state.completing { + a.retentionDirty = true work.state.completing = false } a.mu.Unlock() @@ -363,6 +397,7 @@ func (a *SlotAssembler) processCompletion(ctx context.Context, work *slotComplet if ctx.Err() != nil { return processedSlotCompletion{canceled: true} } + ctx = withEntryPipelineTrace(ctx, work.state.pipelineTrace) startedAt := time.Now() timings := block.TurbineIngressTimings{CompletionQueueDelay: startedAt.Sub(work.queuedAt)} _ = statsd.Duration(statsd.TurbineBlockCompletionQueueDelay, timings.CompletionQueueDelay, nil) @@ -374,19 +409,27 @@ func (a *SlotAssembler) processCompletion(ctx context.Context, work *slotComplet } decodeStartedAt := time.Now() - var decodeTimings entryDecodeTimings + decodeTimings := entryDecodeTimings{ctx: ctx} + if work.state.prefetch != nil { + decodeTimings.prefetched = work.state.prefetch.batches + } blk, parentInfo, roots, err := work.state.decodeBlock(&decodeTimings) decodeTotal := time.Since(decodeStartedAt) - decodeOnly := decodeTotal - decodeTimings.transactionParse + decodeOnly := decodeTotal - decodeTimings.transactionParse - decodeTimings.prefetchWait if decodeOnly < 0 { decodeOnly = 0 } timings.BlockDecode = decodeOnly timings.TransactionParse = decodeTimings.transactionParse + timings.EarlyPreparationWait = decodeTimings.prefetchWait _ = statsd.Duration(statsd.TurbineBlockDecode, timings.BlockDecode, nil) _ = statsd.Duration(statsd.TurbineTransactionParse, timings.TransactionParse, nil) processed := processedSlotCompletion{block: blk, parentInfo: parentInfo, roots: roots, err: err, timings: timings} if err != nil { + if ctx.Err() != nil { + processed.canceled = true + processed.err = nil + } return processed } if ctx.Err() != nil { @@ -395,7 +438,12 @@ func (a *SlotAssembler) processCompletion(ctx context.Context, work *slotComplet } sigverifyStartedAt := time.Now() - processed.err = work.verifyTransactions(ctx, blk) + if work.state.prefetch != nil && len(decodeTimings.retained) > 0 { + processed.err = verifyDecodedEntryBatchesWithTimings(ctx, blk, decodeTimings.retained, work.state.prefetch.pool.verifier, &decodeTimings) + } else { + processed.err = work.verifyTransactions(ctx, blk) + } + earlyEntryTimings(&decodeTimings, work.state.fullAt, &processed.timings) processed.timings.TransactionSigverify = time.Since(sigverifyStartedAt) _ = statsd.Duration(statsd.TurbineTransactionSigverify, processed.timings.TransactionSigverify, nil) if ctx.Err() != nil { @@ -406,6 +454,13 @@ func (a *SlotAssembler) processCompletion(ctx context.Context, work *slotComplet if processed.err == nil { blk.MarkTransactionSignaturesVerified() processed.completionReadyAt = time.Now() + processed.timings.FullToReady = processed.completionReadyAt.Sub(work.state.fullAt) + _ = statsd.Duration(statsd.TurbineFullToReady, processed.timings.FullToReady, nil) + _ = statsd.Duration(statsd.TurbineEarlyPreparationWait, processed.timings.EarlyPreparationWait, nil) + _ = statsd.Duration(statsd.TurbineEarlyTransactionParse, processed.timings.EarlyTransactionParse, nil) + _ = statsd.Duration(statsd.TurbineEarlyTransactionSigverify, processed.timings.EarlyTransactionSigverify, nil) + _ = statsd.Count(statsd.TurbineEarlyVerifiedTransactions, int64(processed.timings.EarlyVerifiedTransactions), nil) + queueEntryPipelineReport(work.state, blk, &decodeTimings, startedAt, processed.completionReadyAt) } return processed } @@ -420,6 +475,13 @@ func (a *SlotAssembler) finalizeCompletion(work *slotCompletionWork, processed p a.mu.Unlock() return nil, nil } + // Every terminal outcome releases this generation's retention protection. + a.retentionDirty = true + if processed.canceled { + state.completing = false + a.mu.Unlock() + return nil, nil + } if processed.err != nil { // Retain a deterministic decode/verification failure on the live full // state so catchup diagnostics report poison instead of a missing slot. @@ -434,6 +496,7 @@ func (a *SlotAssembler) finalizeCompletion(work *slotCompletionWork, processed p if !a.acceptAlpenglowBlockIDLocked(blk) { a.trackNonCanonicalBlockIDLocked(blk) a.recordPartialObsLocked(state) + a.releasePrefetchLocked(state) delete(a.slots, state.slot) a.mu.Unlock() if work.reportNonCanonical { @@ -442,6 +505,7 @@ func (a *SlotAssembler) finalizeCompletion(work *slotCompletionWork, processed p return nil, nil } + a.releasePrefetchLocked(state) delete(a.slots, state.slot) a.completedSlots[state.slot] = struct{}{} a.trackBlockIDLocked(blk) @@ -510,6 +574,9 @@ func (a *SlotAssembler) SetKnownAlpenglowBlockID(slot uint64, blockID solana.Has if _, rejected := a.rejectedBlockIDs[slot][blockID]; rejected { return } + if _, exists := a.knownBlockIDs[slot]; !exists { + a.retentionDirty = true + } a.knownBlockIDs[slot] = blockID } @@ -530,6 +597,7 @@ func (a *SlotAssembler) RejectAlpenglowBlockID(slot uint64, blockID solana.Hash) if ids == nil { ids = make(map[solana.Hash]struct{}) a.rejectedBlockIDs[slot] = ids + a.retentionDirty = true } ids[blockID] = struct{}{} if a.knownBlockIDs[slot] == blockID { @@ -541,7 +609,9 @@ func (a *SlotAssembler) ResetSlot(slot uint64) { a.mu.Lock() defer a.mu.Unlock() + a.retentionDirty = true a.recordPartialObsLocked(a.slots[slot]) + a.releasePrefetchLocked(a.slots[slot]) delete(a.slots, slot) delete(a.completedSlots, slot) } @@ -635,6 +705,32 @@ func (a *SlotAssembler) slotTooOldLocked(slot uint64) bool { } func (a *SlotAssembler) pruneOldSlotsLocked() { + if !a.retentionSwept || a.retentionDirty || a.retentionSweepEdge != a.maxObservedSlot || a.retentionSweepFloor != a.retentionFloor { + a.sweepRetentionMapsLocked() + a.retentionSwept = true + a.retentionDirty = false + a.retentionSweepEdge = a.maxObservedSlot + a.retentionSweepFloor = a.retentionFloor + } + // New incomplete generations can exceed the cap without advancing the + // edge (especially during catch-up). Never cache the capacity check. + + if len(a.slots) == 0 { + return + } + for len(a.slots) > maxRetainedIncompleteSlotCap { + victim, ok := a.capEvictionCandidateLocked() + if !ok { + return + } + a.recordPartialObsLocked(a.slots[victim]) + a.releasePrefetchLocked(a.slots[victim]) + delete(a.slots, victim) + a.evictedSlots++ + } +} + +func (a *SlotAssembler) sweepRetentionMapsLocked() { if len(a.slots) > 0 && a.maxObservedSlot > maxRetainedIncompleteSlotLag { minSlot := a.maxObservedSlot - maxRetainedIncompleteSlotLag if a.retentionFloor > 0 && a.retentionFloor < minSlot { @@ -643,6 +739,7 @@ func (a *SlotAssembler) pruneOldSlotsLocked() { for slot, state := range a.slots { if slot < minSlot && !state.completing { a.recordPartialObsLocked(state) + a.releasePrefetchLocked(a.slots[slot]) delete(a.slots, slot) a.evictedSlots++ } @@ -686,18 +783,6 @@ func (a *SlotAssembler) pruneOldSlotsLocked() { } a.prunePriorityRepairSlotsLocked() - if len(a.slots) == 0 { - return - } - for len(a.slots) > maxRetainedIncompleteSlotCap { - victim, ok := a.capEvictionCandidateLocked() - if !ok { - return - } - a.recordPartialObsLocked(a.slots[victim]) - delete(a.slots, victim) - a.evictedSlots++ - } } // capEvictionCandidateLocked chooses state furthest ahead of replay, rather @@ -1018,6 +1103,9 @@ func (a *SlotAssembler) trackBlockIDLocked(blk *block.Block) { if known, ok := a.knownBlockIDs[blk.Slot]; ok && known != (solana.Hash{}) && known != blockID { return } + if _, exists := a.knownBlockIDs[blk.Slot]; !exists { + a.retentionDirty = true + } a.knownBlockIDs[blk.Slot] = blockID } @@ -1300,21 +1388,48 @@ func (a *SlotAssembler) recoverFEC(state *slotState, fecSetIndex uint32) ([]*Shr } shards[int(layout.dataShreds)+int(pos)] = shard } - encoder, err := a.fecEncoder(layout) - if err != nil { - return nil, err - } - required := make([]bool, int(layout.dataShreds)+int(layout.codingShreds)) var missingData int + missingDataIndex := -1 for idx := 0; idx < int(layout.dataShreds); idx++ { if fec.data[uint32(idx)] == nil { - required[idx] = true missingData++ + missingDataIndex = idx } } if missingData == 0 { return nil, nil } + if missingData == 1 && + layout.dataShreds == rsrecover.DataShards && + layout.codingShreds == rsrecover.CodingShards { + presence, err := rsrecover.Presence(shards) + if err != nil { + return nil, err + } + dst := make([]byte, layout.shardSize) + if err := rsrecover.RecoverOneData(presence, missingDataIndex, shards, dst); err != nil { + return nil, fmt.Errorf("recover one FEC data shred slot %d fec_set=%d: %w", state.slot, fecSetIndex, err) + } + shred, err := fec.recoveredDataShred(uint32(missingDataIndex), dst) + if err != nil { + return nil, err + } + shards[missingDataIndex] = dst + recovered := []*Shred{shred} + if err := a.authenticateRecoveredFEC(fec, shards, recovered); err != nil { + return nil, err + } + return recovered, nil + } + + required := make([]bool, int(layout.dataShreds)+int(layout.codingShreds)) + for idx := 0; idx < int(layout.dataShreds); idx++ { + required[idx] = fec.data[uint32(idx)] == nil + } + encoder, err := a.fecEncoder(layout) + if err != nil { + return nil, err + } if err := encoder.ReconstructSome(shards, required); err != nil { if errors.Is(err, reedsolomon.ErrTooFewShards) { return nil, nil @@ -1337,6 +1452,9 @@ func (a *SlotAssembler) recoverFEC(state *slotState, fecSetIndex uint32) ([]*Shr } recovered = append(recovered, shred) } + if err := a.authenticateRecoveredFEC(fec, shards, recovered); err != nil { + return nil, err + } return recovered, nil } @@ -1498,6 +1616,24 @@ func (s *slotState) complete() bool { } func (s *slotState) orderedShreds() []*Shred { + // Normal completed slots contain exactly the contiguous range 0..lastIndex. + // Keep the sparse fallback: malformed tails and focused partial-state callers + // must not silently lose shreds beyond the last-in-slot marker. + if s.haveLast && uint64(len(s.shreds)) == uint64(s.lastIndex)+1 { + out := make([]*Shred, len(s.shreds)) + for idx := range out { + shred := s.shreds[uint32(idx)] + if shred == nil || shred.Index != uint32(idx) { + return s.sortedShreds() + } + out[idx] = shred + } + return out + } + return s.sortedShreds() +} + +func (s *slotState) sortedShreds() []*Shred { indexes := make([]int, 0, len(s.shreds)) for idx := range s.shreds { indexes = append(indexes, int(idx)) @@ -1507,6 +1643,7 @@ func (s *slotState) orderedShreds() []*Shred { for _, idx := range indexes { out = append(out, s.shreds[uint32(idx)]) } + sort.Slice(out, func(i, j int) bool { return out[i].Index < out[j].Index }) return out } @@ -1514,7 +1651,7 @@ func (s *slotState) orderedShreds() []*Shred { // transaction signature verification. Parent/child identity hints are applied // later under the assembler lock so hints learned while this runs still win. func (s *slotState) decodeBlock(timings *entryDecodeTimings) (*block.Block, *AlpenglowParentInfo, []solana.Hash, error) { - entries, parentInfo, footer, err := decodeEntriesAndAlpenglowMarkersFromDataShreds(s.orderedShreds(), timings) + entries, parentInfo, footer, err := decodeEntriesFromOrderedDataShreds(s.orderedShreds(), timings) if err != nil { return nil, nil, nil, err } @@ -1660,35 +1797,36 @@ func (s *slotState) fecSetMerkleRoots() ([]solana.Hash, error) { } func (f *fecState) merkleRoot() (solana.Hash, bool, error) { - for _, idx := range sortedUint32Keys(f.data) { - shred := f.data[idx] - if shred == nil || shred.Recovered { - continue - } - root, err := shred.MerkleRoot() - if err != nil { - if errors.Is(err, ErrUnsupportedShred) { - continue + // Preserve the old lowest-data-index, then lowest-coding-position choice, + // including its first error. Unsupported variants and recovered data have + // no usable proof. Selecting the minimum needs neither sorting nor scratch. + selected := f.data[0] + var first uint32 + // Relative index zero is already the minimum. Repair or legacy/malformed + // inputs may lack that proof and still take the general selection path. + if !hasMerkleRootProof(selected) || selected.Recovered { + selected = nil + for idx, shred := range f.data { + if hasMerkleRootProof(shred) && !shred.Recovered && (selected == nil || idx < first) { + selected, first = shred, idx } - return solana.Hash{}, false, err } - return root, true, nil } - for _, pos := range sortedUint16Keys(f.coding) { - shred := f.coding[pos] - if shred == nil { - continue - } - root, err := shred.MerkleRoot() - if err != nil { - if errors.Is(err, ErrUnsupportedShred) { - continue + if selected == nil { + for pos, shred := range f.coding { + if hasMerkleRootProof(shred) && (selected == nil || uint32(pos) < first) { + selected, first = shred, uint32(pos) } - return solana.Hash{}, false, err } - return root, true, nil } - return solana.Hash{}, false, nil + if selected == nil { + return solana.Hash{}, false, nil + } + if cached := f.rootCache; cached != nil && cached.matches(selected) { + return cached.root, true, nil + } + root, err := selected.MerkleRoot() + return root, err == nil, err } func merkleTreeRoot(leaves []solana.Hash) solana.Hash { diff --git a/pkg/turbine/assembler_test.go b/pkg/turbine/assembler_test.go index f70f3d695..e38b468b3 100644 --- a/pkg/turbine/assembler_test.go +++ b/pkg/turbine/assembler_test.go @@ -622,6 +622,15 @@ func TestDecodeAlpenglowParentMarkers(t *testing.T) { func TestSlotAssemblerRecoversMissingMerkleDataShredFromCodingShreds(t *testing.T) { dataShreds := localnetMerkleShreds(t, "d") codeShreds := localnetMerkleShreds(t, "c") + // These 2022 fixtures supply a useful non-power-of-two, unchained 1+17 + // erasure layout, but their old proofs do not yield a common root under + // today's Merkle hashing. Re-sign each complete tree using current proofs; + // recovery must now authenticate the tree, not just reconstruct the data. + for i, data := range dataShreds { + packets := append([][]byte{data}, codeShreds[i*17:(i+1)*17]...) + resignRecoveryFixture(t, packets) + } + if len(dataShreds) < 2 || len(codeShreds) == 0 { t.Fatalf("fixture needs data and coding shreds") } diff --git a/pkg/turbine/broadcast.go b/pkg/turbine/broadcast.go index fee229158..814951132 100644 --- a/pkg/turbine/broadcast.go +++ b/pkg/turbine/broadcast.go @@ -3,7 +3,6 @@ package turbine import ( "fmt" "net" - "sort" "sync" "github.com/gagliardetto/solana-go" @@ -116,8 +115,8 @@ type BroadcastSessionConfig struct { // It seeds the chained merkle root embedded in this slot's first FEC batch. ParentChainedMerkleRoot solana.Hash Broadcaster PacketBroadcaster - UserAgent []byte - Version uint16 + UserAgent []byte + Version uint16 } func NewBroadcastSession(cfg BroadcastSessionConfig) *BroadcastSession { @@ -192,7 +191,7 @@ func (s *BroadcastSession) broadcastComponent(component BlockComponent, isLastIn if s.broadcaster == nil { return nil } - batch, nextData, nextCode, err := s.shredder.MakeMerkleShredsFromComponent( + batch, nextData, nextCode, err := s.shredder.makeMerklePacketsFromComponent( s.leader, component, isLastInSlot, @@ -203,44 +202,9 @@ func (s *BroadcastSession) broadcastComponent(component BlockComponent, isLastIn if err != nil { return err } - s.chainedMerkleRoot = batch.ChainedMerkleRoot + s.chainedMerkleRoot = batch.chainedMerkleRoot s.nextDataIndex = nextData s.nextCodeIndex = nextCode - if len(batch.DataShreds) > 0 { - s.fecSetRoots = appendFECSetMerkleRoots(s.fecSetRoots, batch.DataShreds) - } - return s.broadcaster.Broadcast(batch.Packets) -} - -func appendFECSetMerkleRoots(roots []solana.Hash, dataShreds []*Shred) []solana.Hash { - if len(dataShreds) == 0 { - return roots - } - indices := make([]uint32, 0) - seen := make(map[uint32]struct{}) - for _, shred := range dataShreds { - if shred == nil { - continue - } - if _, ok := seen[shred.FECSetIndex]; ok { - continue - } - seen[shred.FECSetIndex] = struct{}{} - indices = append(indices, shred.FECSetIndex) - } - sort.Slice(indices, func(i, j int) bool { return indices[i] < indices[j] }) - for _, fecSetIndex := range indices { - for _, shred := range dataShreds { - if shred == nil || shred.FECSetIndex != fecSetIndex { - continue - } - root, err := shred.MerkleRoot() - if err != nil { - continue - } - roots = append(roots, root) - break - } - } - return roots + s.fecSetRoots = append(s.fecSetRoots, batch.fecSetRoots...) + return s.broadcaster.Broadcast(batch.packets) } diff --git a/pkg/turbine/broadcast_test.go b/pkg/turbine/broadcast_test.go index 09e82a397..11c211c76 100644 --- a/pkg/turbine/broadcast_test.go +++ b/pkg/turbine/broadcast_test.go @@ -52,6 +52,70 @@ func TestBroadcastSessionHeaderAndFooter(t *testing.T) { require.NotEqual(t, parentBlockID, chainedRoot) } +func TestBroadcastSessionMatchesParsedShreds(t *testing.T) { + leader := testBroadcastLeader(t) + parentID, parentRoot := solana.Hash{0xaa}, solana.Hash{0xbb} + capture := &packetCapture{} + session := NewBroadcastSession(BroadcastSessionConfig{ + Leader: leader, Slot: 100, ParentSlot: 99, Version: 7, + ParentChainedMerkleRoot: parentRoot, Broadcaster: capture, + }) + shredder := Shredder{Slot: 100, ParentSlot: 99, Version: 7} + txns := make([]solana.Transaction, 400) + for i := range txns { + txns[i] = mustParseTransferTx(t, uint64(i)) + } + entries, err := NewEntryBatch([]Entry{{NumHashes: 1, Hash: solana.Hash{1}, Txns: txns}}) + require.NoError(t, err) + tick, err := NewEntryBatch([]Entry{{NumHashes: 1, Hash: solana.Hash{2}}}) + require.NoError(t, err) + components := []BlockComponent{ + NewBlockHeader(99, parentID), + entries, // More than two FEC sets: every root must enter the block ID. + NewUpdateParent(98, solana.Hash{3}), + NewBlockFooter(BlockFooter{BankHash: solana.Hash{4}}), + tick, + } + var nextData, nextCode uint32 + root := parentRoot + var roots []solana.Hash + for i, component := range components { + last := i == len(components)-1 + batch, data, code, err := shredder.MakeMerkleShredsFromComponent( + leader, component, last, root, nextData, nextCode, + ) + require.NoError(t, err) + if i == 1 { + require.Greater(t, len(batch.DataShreds), 2*dataShredsPerFECBlock) + } + for j, shred := range batch.DataShreds { + if j == 0 || shred.FECSetIndex != batch.DataShreds[j-1].FECSetIndex { + fecRoot, err := shred.MerkleRoot() + require.NoError(t, err) + roots = append(roots, fecRoot) + } + } + capture.packets = nil + require.NoError(t, session.BroadcastComponent(component, last)) + require.Equal(t, batch.Packets, capture.packets) + require.Equal(t, batch.ChainedMerkleRoot, session.ChainedMerkleRoot()) + require.Equal(t, data, session.nextDataIndex) + require.Equal(t, code, session.nextCodeIndex) + require.Equal(t, roots, session.fecSetRoots) + require.Equal(t, DoubleMerkleBlockID(99, parentID, roots), session.BlockID(99, parentID)) + root, nextData, nextCode = batch.ChainedMerkleRoot, data, code + } + + // Invalid components must not publish packets or advance the commitment. + capture.packets = nil + require.Error(t, session.BroadcastComponent(BlockComponent{Marker: &BlockMarker{Kind: 255}}, false)) + require.Empty(t, capture.packets) + require.Equal(t, root, session.ChainedMerkleRoot()) + require.Equal(t, nextData, session.nextDataIndex) + require.Equal(t, nextCode, session.nextCodeIndex) + require.Equal(t, roots, session.fecSetRoots) +} + func TestUDPBroadcasterLoopback(t *testing.T) { recvAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0") require.NoError(t, err) diff --git a/pkg/turbine/cancellation_regression_test.go b/pkg/turbine/cancellation_regression_test.go index 1f85e4df9..719931216 100644 --- a/pkg/turbine/cancellation_regression_test.go +++ b/pkg/turbine/cancellation_regression_test.go @@ -17,86 +17,52 @@ func TestTransactionVerifierCancellationDuringBlockedAdmissionJoinsAdmittedJobs( const workers = 2 blocker := verifierTestBlock(workers) target := verifierTestBlock(2 * workers) - filler := &solana.Transaction{} - release := make(chan struct{}) var releaseOnce sync.Once started := make(chan struct{}, workers) var targetFirstCalls atomic.Int32 var targetLaterCalls atomic.Int32 - - verifier := newTransactionVerifier(workers, workers, func(tx *solana.Transaction) error { + // Single-transaction groups make the blocked-admission boundary exact. + verifier := newTransactionVerifierWithBatchTarget(workers, 1, 1, func(tx *solana.Transaction) error { switch tx { case blocker.Transactions[0], blocker.Transactions[1]: started <- struct{}{} <-release case target.Transactions[0]: targetFirstCalls.Add(1) - case target.Transactions[1], target.Transactions[2], target.Transactions[3]: + default: targetLaterCalls.Add(1) } return nil }) - + defer verifier.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() blockerDone := make(chan error, 1) targetDone := make(chan error, 1) - var calls sync.WaitGroup - calls.Add(1) - go func() { - defer calls.Done() - blockerDone <- verifier.verifyBlock(blocker) - }() + go func() { blockerDone <- verifier.verifyBlock(blocker) }() for range workers { - select { - case <-started: - case <-time.After(3 * time.Second): - cancel() - releaseOnce.Do(func() { close(release) }) - calls.Wait() - verifier.closeAndWait() - t.Fatal("timed out occupying transaction verifier workers") - } + waitSignal(t, started, "occupied verifier worker") } - // Leave one queued job ahead of the target. With both workers occupied and - // a two-entry queue, the target admits transaction 0 and then blocks trying - // to admit transaction 1. Cancellation must wait for transaction 0 to drain, - // while transactions 1 and all later chunks must never reach a worker. - var fillerErr error - var fillerDone sync.WaitGroup - fillerDone.Add(1) - verifier.jobs <- transactionVerifyJob{tx: filler, err: &fillerErr, done: &fillerDone} - - calls.Add(1) - go func() { - defer calls.Done() - targetDone <- verifier.verifyBlockContext(ctx, target) - }() + // Both workers are occupied. The target's first group fills the one-entry + // queue, and its second blocks on admission. Cancellation must still join + // the first group while preventing any later transactions from running. + go func() { targetDone <- verifier.verifyBlockContext(ctx, target) }() deadline := time.Now().Add(3 * time.Second) for len(verifier.jobs) != cap(verifier.jobs) { if time.Now().After(deadline) { - cancel() - releaseOnce.Do(func() { close(release) }) - calls.Wait() - fillerDone.Wait() - verifier.closeAndWait() - t.Fatalf("transaction queue did not fill: len=%d cap=%d", len(verifier.jobs), cap(verifier.jobs)) + t.Fatal("transaction queue did not fill") } time.Sleep(time.Millisecond) } - cancel() select { case err := <-targetDone: - releaseOnce.Do(func() { close(release) }) - calls.Wait() - fillerDone.Wait() - verifier.closeAndWait() t.Fatalf("canceled verifier returned before its admitted job joined: %v", err) case <-time.After(50 * time.Millisecond): } - releaseOnce.Do(func() { close(release) }) select { case err := <-targetDone: @@ -114,13 +80,6 @@ func TestTransactionVerifierCancellationDuringBlockedAdmissionJoinsAdmittedJobs( case <-time.After(3 * time.Second): t.Fatal("blocker verification did not drain") } - fillerDone.Wait() - calls.Wait() - verifier.closeAndWait() - - if fillerErr != nil { - t.Fatalf("filler verification: %v", fillerErr) - } if got := targetFirstCalls.Load(); got != 1 { t.Fatalf("admitted target transaction calls = %d, want 1", got) } diff --git a/pkg/turbine/cluster_nodes.go b/pkg/turbine/cluster_nodes.go index 8446a3966..ff6e0bdc0 100644 --- a/pkg/turbine/cluster_nodes.go +++ b/pkg/turbine/cluster_nodes.go @@ -89,6 +89,13 @@ func newClusterNodes(cfg ClusterNodesConfig, broadcast bool) *ClusterNodes { // shuffle because it already owns the shred. Tree placement and fanout match // Agave ClusterNodes::get_retransmit_addrs. func (c *ClusterNodes) RetransmitPeers(leader solana.PublicKey, shred ShredID, fanout int) (uint8, []*net.UDPAddr, error) { + return c.retransmitPeersInto(leader, shred, fanout, nil) +} + +// retransmitPeersInto uses caller-owned result storage. The caller must finish +// using the returned slice before reusing dst; addresses still belong to this +// immutable cluster snapshot. Public callers retain the allocating API above. +func (c *ClusterNodes) retransmitPeersInto(leader solana.PublicKey, shred ShredID, fanout int, dst []*net.UDPAddr) (uint8, []*net.UDPAddr, error) { if c == nil || fanout <= 0 { return maxTurbineHops - 1, nil, nil } @@ -123,7 +130,10 @@ func (c *ClusterNodes) RetransmitPeers(leader solana.PublicKey, shred ShredID, f step = 1 } position := anchor*fanout + offset + 1 - peers := make([]*net.UDPAddr, 0, fanout) + peers := dst[:0] + if dst == nil { + peers = make([]*net.UDPAddr, 0, fanout) + } shufflePosition := selfPos for range fanout { var index int diff --git a/pkg/turbine/completion_order_root_test.go b/pkg/turbine/completion_order_root_test.go new file mode 100644 index 000000000..5fd8fd7f7 --- /dev/null +++ b/pkg/turbine/completion_order_root_test.go @@ -0,0 +1,228 @@ +package turbine + +import ( + "bytes" + "context" + "errors" + "sort" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestCompletedShredOrderPreservesSparseAndExtraTails(t *testing.T) { + for _, indexes := range [][]uint32{{0, 1, 2}, {0, 2, 3}, {0, 1, 2, 9}, {1, 3, 7}} { + s := &slotState{haveLast: true, lastIndex: 2, shreds: make(map[uint32]*Shred)} + for _, idx := range indexes { + s.shreds[idx] = &Shred{Index: idx, Type: ShredTypeData} + } + got := s.orderedShreds() + require.Len(t, got, len(indexes)) + for i, idx := range indexes { + require.Same(t, s.shreds[idx], got[i]) + } + } +} + +func TestOrderedEntryDecodeMatchesPublicUnorderedDecode(t *testing.T) { + packets := agavePaddedSlot1752420Packets(t) + s := &slotState{shreds: make(map[uint32]*Shred)} + var shuffled []*Shred + for _, packet := range packets { + shred, err := ParseShred(packet) + require.NoError(t, err) + s.shreds[shred.Index] = shred + shuffled = append(shuffled, shred) + } + for i, j := 0, len(shuffled)-1; i < j; i, j = i+1, j-1 { + shuffled[i], shuffled[j] = shuffled[j], shuffled[i] + } + want, parent, footer, err := DecodeEntriesAndAlpenglowMarkersFromDataShreds(shuffled) + require.NoError(t, err) + got, gotParent, gotFooter, err := decodeEntriesFromOrderedDataShreds(s.orderedShreds(), nil) + require.NoError(t, err) + require.Equal(t, want, got) + require.Equal(t, parent, gotParent) + require.Equal(t, footer, gotFooter) +} + +func TestAuthenticatedFECRootRejectsChangedInputAndReplacement(t *testing.T) { + shred, leader := buildSignedTestShred(t, 100, 42) + var verifier shredSigCache + root, err := verifier.verifyShredRoot(shred, leader) + require.NoError(t, err) + f := &fecState{data: map[uint32]*Shred{0: shred}} + f.rememberAuthenticatedRoot(shred, root) + cached := f.rootCache + require.True(t, cached.matches(shred)) + for i := range shred.Payload { + shred.Payload[i] ^= 1 + require.False(t, cached.matches(shred), "payload byte %d", i) + shred.Payload[i] ^= 1 + } + for _, mutate := range []func(*Shred){ + func(s *Shred) { s.Variant ^= 1 }, func(s *Shred) { s.Type = ShredTypeCode }, + func(s *Shred) { s.Index++ }, func(s *Shred) { s.FECSetIndex++ }, + func(s *Shred) { s.NumDataShreds++ }, func(s *Shred) { s.Position++ }, + } { + original := *shred + mutate(shred) + require.False(t, cached.matches(shred)) + *shred = original + } + shred.Payload[dataHeaderSize] ^= 1 + want, err := shred.MerkleRoot() + require.NoError(t, err) + got, ok, err := f.merkleRoot() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, want, got) + require.NotEqual(t, root, got) + _, err = verifier.verifyShredRoot(shred, leader) + require.ErrorIs(t, err, ErrInvalidSignature) + shred.Payload[dataHeaderSize] ^= 1 + replacement := *shred + require.False(t, cached.matches(&replacement)) + f.data[0] = &replacement + got, ok, err = f.merkleRoot() + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, root, got) +} + +func TestAuthenticatedFECRootAdmissionAndReset(t *testing.T) { + shred, leader := buildSignedTestShred(t, 100, 42) + var verifier shredSigCache + root, err := verifier.verifyShredRoot(shred, leader) + require.NoError(t, err) + a := NewSlotAssembler() + work, err := a.addShredFromWithRoot(shred, false, &root) + require.NoError(t, err) + require.NotNil(t, work) + f := work.state.fecSets[0] + require.True(t, f.rootCache.matches(shred)) + // A duplicate cannot overwrite the admitted root, even while completion is + // retried after cancellation. Reset starts a separate FEC generation. + a.abortCompletion(work) + wrong := solana.Hash{99} + _, err = a.addShredFromWithRoot(shred, true, &wrong) + require.NoError(t, err) + require.Equal(t, root, f.rootCache.root) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.True(t, a.processCompletion(ctx, work).canceled) + a.ResetSlot(100) + work, err = a.addShredFrom(shred, false) + require.NoError(t, err) + require.NotNil(t, work) + require.Nil(t, work.state.fecSets[0].rootCache) +} + +func referenceFECRoot(f *fecState) (solana.Hash, bool, error) { + for _, idx := range sortedUint32Keys(f.data) { + s := f.data[idx] + if s == nil || s.Recovered { + continue + } + root, err := s.MerkleRoot() + if errors.Is(err, ErrUnsupportedShred) { + continue + } + return root, err == nil, err + } + for _, idx := range sortedUint16Keys(f.coding) { + s := f.coding[idx] + if s == nil { + continue + } + root, err := s.MerkleRoot() + if errors.Is(err, ErrUnsupportedShred) { + continue + } + return root, err == nil, err + } + return solana.Hash{}, false, nil +} + +func FuzzFECRootSelectionMatchesSortedReference(f *testing.F) { + f.Add([]byte{0, 1, 2, 3, 4, 5, 6, 7}) + f.Add([]byte{9, 0, 0, 0, 2, 1, 3, 7}) + f.Fuzz(func(t *testing.T, choices []byte) { + if len(choices) > 512 { + t.Skip() + } + state := &fecState{data: make(map[uint32]*Shred), coding: make(map[uint16]*Shred)} + for i, choice := range choices { + s := &Shred{Variant: merkleDataVariant, Type: ShredType(choice % 3), Payload: bytes.Repeat([]byte{choice}, dataPayloadSize)} + switch choice % 5 { + case 0: + s = nil + case 1: + s.Recovered = true + case 2: + s.Variant = legacyDataVariant + case 3: + s.Payload = s.Payload[:20] + } + if i%2 == 0 { + state.data[uint32(choice)] = s + } else { + state.coding[uint16(choice)] = s + } + } + want, wantOK, wantErr := referenceFECRoot(state) + got, gotOK, gotErr := state.merkleRoot() + require.Equal(t, want, got) + require.Equal(t, wantOK, gotOK) + if wantErr == nil { + require.NoError(t, gotErr) + } else { + require.EqualError(t, gotErr, wantErr.Error()) + } + }) +} + +func TestFECRootCacheKeepsDeterministicDataPrecedence(t *testing.T) { + packets := append(localnetMerkleShreds(t, "d"), localnetMerkleShreds(t, "c")...) + f := &fecState{data: make(map[uint32]*Shred), coding: make(map[uint16]*Shred)} + var shreds []*Shred + for _, packet := range packets { + s, err := ParseShred(packet) + require.NoError(t, err) + if s.FECSetIndex != 0 { + continue + } + shreds = append(shreds, s) + } + require.NotEmpty(t, shreds) + sort.Slice(shreds, func(i, j int) bool { + if shreds[i].Type != shreds[j].Type { + return shreds[i].Type == ShredTypeCode + } + return shreds[i].Index > shreds[j].Index + }) + for _, s := range shreds { + root, err := s.MerkleRoot() + require.NoError(t, err) + if s.Type == ShredTypeData { + f.data[s.Index-s.FECSetIndex] = s + } else { + f.coding[s.Position] = s + } + f.rememberAuthenticatedRoot(s, root) + want, wantOK, wantErr := referenceFECRoot(f) + got, gotOK, gotErr := f.merkleRoot() + require.Equal(t, wantErr, gotErr) + require.Equal(t, wantOK, gotOK) + require.Equal(t, want, got) + } + for _, s := range f.data { + s.Recovered = true + } + want, wantOK, wantErr := referenceFECRoot(f) + got, gotOK, gotErr := f.merkleRoot() + require.Equal(t, wantErr, gotErr) + require.Equal(t, wantOK, gotOK) + require.Equal(t, want, got) +} diff --git a/pkg/turbine/component_shredder.go b/pkg/turbine/component_shredder.go index ad4c9229b..109408a4c 100644 --- a/pkg/turbine/component_shredder.go +++ b/pkg/turbine/component_shredder.go @@ -14,7 +14,7 @@ type Shredder struct { ReferenceTick uint8 } -// ShredBatch is one FEC batch emitted for a single block component. +// ShredBatch contains the FEC batches emitted for a single block component. type ShredBatch struct { Slot uint64 Component BlockComponent @@ -34,23 +34,8 @@ func (s *Shredder) MakeMerkleShredsFromComponent( nextShredIndex uint32, nextCodeIndex uint32, ) (ShredBatch, uint32, uint32, error) { - bytes, err := MarshalBlockComponent(component) - if err != nil { - return ShredBatch{}, nextShredIndex, nextCodeIndex, err - } - gen := ShredGenerator{ - Slot: s.Slot, - ParentSlot: s.ParentSlot, - Version: s.Version, - ReferenceTick: s.ReferenceTick, - } - packets, root, nextData, nextCode, err := gen.MakeShredsFromData( - leader, - bytes, - isLastInSlot, - chainedMerkleRoot, - nextShredIndex, - nextCodeIndex, + generated, nextData, nextCode, err := s.makeMerklePacketsFromComponent( + leader, component, isLastInSlot, chainedMerkleRoot, nextShredIndex, nextCodeIndex, ) if err != nil { return ShredBatch{}, nextShredIndex, nextCodeIndex, err @@ -58,11 +43,11 @@ func (s *Shredder) MakeMerkleShredsFromComponent( batch := ShredBatch{ Slot: s.Slot, Component: component, - Packets: packets, - ChainedMerkleRoot: root, + Packets: generated.packets, + ChainedMerkleRoot: generated.chainedMerkleRoot, IsLastInSlot: isLastInSlot, } - for _, packet := range packets { + for _, packet := range batch.Packets { shred, err := ParseShred(packet) if err != nil { return ShredBatch{}, nextShredIndex, nextCodeIndex, fmt.Errorf("parse generated shred: %w", err) @@ -75,3 +60,37 @@ func (s *Shredder) MakeMerkleShredsFromComponent( } return batch, nextData, nextCode, nil } + +// makeMerklePacketsFromComponent serves the producer, which needs the wire +// packets and FEC roots but not the owning Shred objects exposed by the public API. +func (s *Shredder) makeMerklePacketsFromComponent( + leader solana.PrivateKey, + component BlockComponent, + isLastInSlot bool, + chainedMerkleRoot solana.Hash, + nextShredIndex uint32, + nextCodeIndex uint32, +) (shredPackets, uint32, uint32, error) { + bytes, err := MarshalBlockComponent(component) + if err != nil { + return shredPackets{}, nextShredIndex, nextCodeIndex, err + } + gen := ShredGenerator{ + Slot: s.Slot, + ParentSlot: s.ParentSlot, + Version: s.Version, + ReferenceTick: s.ReferenceTick, + } + batch, nextData, nextCode, err := gen.makeShredsFromData( + leader, + bytes, + isLastInSlot, + chainedMerkleRoot, + nextShredIndex, + nextCodeIndex, + ) + if err != nil { + return shredPackets{}, nextShredIndex, nextCodeIndex, err + } + return batch, nextData, nextCode, nil +} diff --git a/pkg/turbine/component_test.go b/pkg/turbine/component_test.go index 64b05760c..2617715be 100644 --- a/pkg/turbine/component_test.go +++ b/pkg/turbine/component_test.go @@ -131,6 +131,36 @@ func TestShredEntryBatchRoundTrip(t *testing.T) { require.Equal(t, entry.NumHashes, components[0].EntryBatch[0].NumHashes) } +func TestShredMultiFECEntryBatchRoundTrip(t *testing.T) { + leader := testLeader(t) + entries := make([]turbine.Entry, 1300) + for i := range entries { + entries[i] = turbine.Entry{NumHashes: 1, Hash: solana.Hash{byte(i), byte(i >> 8)}} + } + component, err := turbine.NewEntryBatch(entries) + require.NoError(t, err) + + shredder := turbine.Shredder{Slot: 100, ParentSlot: 99, Version: 42, ReferenceTick: 63} + batch, _, _, err := shredder.MakeMerkleShredsFromComponent( + leader, component, true, solana.Hash{}, 0, 0, + ) + require.NoError(t, err) + require.Greater(t, len(batch.DataShreds), 32) + for i, shred := range batch.DataShreds[:len(batch.DataShreds)-1] { + require.False(t, shred.DataComplete(), "intermediate data shred %d ended the component", i) + } + require.True(t, batch.DataShreds[len(batch.DataShreds)-1].DataComplete()) + require.True(t, batch.DataShreds[len(batch.DataShreds)-1].LastInSlot()) + + components, err := turbine.DecodeComponentsFromDataShreds(batch.DataShreds) + require.NoError(t, err) + require.Len(t, components, 1) + require.Len(t, components[0].EntryBatch, len(entries)) + for i := range entries { + require.Equal(t, entries[i].Hash, components[0].EntryBatch[i].Hash) + } +} + func TestShredBlockHeaderMarkerRoundTrip(t *testing.T) { leader := testLeader(t) parentID := solana.Hash{8} diff --git a/pkg/turbine/entries.go b/pkg/turbine/entries.go index 9f6251318..e9372204d 100644 --- a/pkg/turbine/entries.go +++ b/pkg/turbine/entries.go @@ -1,6 +1,8 @@ package turbine import ( + "bytes" + "context" "encoding/binary" "fmt" "sort" @@ -40,6 +42,12 @@ type AlpenglowParentInfo struct { type entryDecodeTimings struct { transactionParse time.Duration + ctx context.Context + prefetched map[uint32]*prefetchedShredBatch + retained []*prefetchedShredBatch + all []*prefetchedShredBatch + prefetchWait time.Duration + traceFallback *transactionVerification } func (e *Entry) UnmarshalWithDecoder(decoder *bin.Decoder) error { @@ -124,69 +132,109 @@ func decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds []*Shred, timings *en sort.Slice(shreds, func(i, j int) bool { return shreds[i].Index < shreds[j].Index }) + return decodeEntriesFromOrderedDataShreds(shreds, timings) +} - type decodedEntryBatch struct { - start uint32 - entries []Entry - } - var entryBatches []decodedEntryBatch +// decodeEntriesFromOrderedDataShreds requires increasing shred indexes. The +// assembler supplies that order directly; public decoding still sorts input. +func decodeEntriesFromOrderedDataShreds(shreds []*Shred, timings *entryDecodeTimings) ([]Entry, *AlpenglowParentInfo, *BlockFooter, error) { + var entryBatches []*prefetchedShredBatch var parentInfo *AlpenglowParentInfo var blockFooter *BlockFooter - var batchBytes []byte var batchStart uint32 + var batchStartPos, batchSize int var haveBatch bool - for _, shred := range shreds { + for shredPos, shred := range shreds { if shred == nil || shred.Type != ShredTypeData { continue } if !haveBatch { batchStart = shred.Index + batchStartPos = shredPos haveBatch = true } - batchBytes = append(batchBytes, shred.Data...) + batchSize += len(shred.Data) if !shred.DataComplete() { continue } - if parent, footer, ok, err := decodeAlpenglowMarkerFromShredBatch(batchBytes, batchStart); err != nil { - return nil, nil, nil, fmt.Errorf("decode alpenglow block marker ending at shred %d: %w", shred.Index, err) - } else if ok { - if parent != nil { - parentInfo, err = mergeAlpenglowParentInfo(parentInfo, parent) - if err != nil { - return nil, nil, nil, fmt.Errorf("merge alpenglow parent marker ending at shred %d: %w", shred.Index, err) + batchShreds := shreds[batchStartPos : shredPos+1] + var batch *prefetchedShredBatch + if timings != nil { + if cached := timings.prefetched[batchStart]; cached != nil && cached.start == batchStart && cached.end == shred.Index { + ctx := timings.ctx + if ctx == nil { + ctx = context.Background() + } + if cached.ready != nil { + waitStarted := time.Now() + select { + case <-cached.ready: + case <-ctx.Done(): + timings.prefetchWait += time.Since(waitStarted) + return nil, nil, nil, ctx.Err() + } + timings.prefetchWait += time.Since(waitStarted) + } + if err := ctx.Err(); err != nil { + return nil, nil, nil, err + } + // Bounds alone cannot prove identity after repair or replacement. + // Read decoded fields only after the preparation channel closes. + // Compare the original slices directly: a cache hit needs no + // second component buffer or copies of already decoded bytes. + if len(cached.raw) == batchSize && dataShredBatchMatches(batchShreds, cached.raw) { + batch = cached } } - if footer != nil { - blockFooter = footer + } + if batch == nil { + // A miss owns a fresh, exactly sized backing array. Transactions + // retain instruction-data slices into it after this call returns. + var traceStart int64 + if timings != nil && entryTraceContext(timings.ctx) { + traceStart = entryTraceNow() + } + batchBytes := make([]byte, 0, batchSize) + for _, part := range batchShreds { + if part != nil && part.Type == ShredTypeData { + batchBytes = append(batchBytes, part.Data...) + } + } + batch = decodeClosedShredBatch(batchBytes, batchStart, shred.Index) + if traceStart != 0 { + batch.traceDecodeStart, batch.traceDecodeEnd = traceStart, entryTraceNow() + } + if timings != nil { + timings.transactionParse += batch.parseDuration } - batchBytes = nil - haveBatch = false - continue } - parseStart := time.Now() - batchEntries, consumed, err := decodeEntryBatchPrefix(batchBytes) if timings != nil { - timings.transactionParse += time.Since(parseStart) + timings.all = append(timings.all, batch) } - // A zero entry count with more bytes denotes a marker. Preserve the - // rejection of unrecognized or misplaced markers in this fallback path. - if err == nil && len(batchEntries) == 0 && consumed != len(batchBytes) { - err = fmt.Errorf("entry batch has %d trailing bytes", len(batchBytes)-consumed) + if batch.err != nil { + return nil, nil, nil, batch.err } - if err != nil { - return nil, nil, nil, fmt.Errorf("decode entry batch ending at shred %d: %w", shred.Index, err) + if batch.marker { + if batch.parent != nil { + var err error + parentInfo, err = mergeAlpenglowParentInfo(parentInfo, batch.parent) + if err != nil { + return nil, nil, nil, fmt.Errorf("merge alpenglow parent marker ending at shred %d: %w", shred.Index, err) + } + } + if batch.footer != nil { + blockFooter = batch.footer + } + batchSize = 0 + haveBatch = false + continue } - entryBatches = append(entryBatches, decodedEntryBatch{ - start: batchStart, - entries: batchEntries, - }) - // Decoded transactions retain slices into the batch buffer for instruction data. - // Keep the backing array alive instead of reusing and overwriting it. - batchBytes = nil + entryBatches = append(entryBatches, batch) + batchSize = 0 haveBatch = false } - if len(batchBytes) != 0 { - return nil, nil, nil, fmt.Errorf("slot ended with %d undecoded entry bytes", len(batchBytes)) + if batchSize != 0 { + return nil, nil, nil, fmt.Errorf("slot ended with %d undecoded entry bytes", batchSize) } var entries []Entry for _, batch := range entryBatches { @@ -199,10 +247,60 @@ func decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds []*Shred, timings *en continue } entries = append(entries, batch.entries...) + if timings != nil { + timings.retained = append(timings.retained, batch) + } } return entries, parentInfo, blockFooter, nil } +// dataShredBatchMatches is equivalent to comparing raw with the concatenated +// data bytes, including all padding. Neither equal prefixes nor changed lengths +// can reuse a cached signature verdict. It does not retain or allocate buffers. +func dataShredBatchMatches(shreds []*Shred, raw []byte) bool { + offset := 0 + for _, shred := range shreds { + if shred == nil || shred.Type != ShredTypeData { + continue + } + if len(shred.Data) > len(raw)-offset || !bytes.Equal(shred.Data, raw[offset:offset+len(shred.Data)]) { + return false + } + offset += len(shred.Data) + } + return offset == len(raw) +} + +// decodeClosedShredBatch owns raw through the returned decoded transactions. +// It decodes the same padded component envelope for early and full assembly; +// marker merging and UpdateParent selection still require the full slot. +func decodeClosedShredBatch(raw []byte, start, end uint32) *prefetchedShredBatch { + batch := &prefetchedShredBatch{start: start, end: end, raw: raw} + parent, footer, marker, err := decodeAlpenglowMarkerFromShredBatch(raw, start) + if err != nil { + batch.err = fmt.Errorf("decode alpenglow block marker ending at shred %d: %w", end, err) + return batch + } + if marker { + batch.parent, batch.footer, batch.marker = parent, footer, true + return batch + } + parseStarted := time.Now() + entries, consumed, err := decodeEntryBatchPrefix(raw) + batch.parseDuration = time.Since(parseStarted) + // A zero entry count with more bytes denotes a marker. Preserve rejection + // of unknown or misplaced markers instead of treating them as an empty batch. + if err == nil && len(entries) == 0 && consumed != len(raw) { + err = fmt.Errorf("entry batch has %d trailing bytes", len(raw)-consumed) + } + if err != nil { + batch.err = fmt.Errorf("decode entry batch ending at shred %d: %w", end, err) + return batch + } + batch.entries = entries + return batch +} + func decodeEntryBatch(data []byte) ([]Entry, error) { entries, consumed, err := decodeEntryBatchPrefix(data) if err != nil { diff --git a/pkg/turbine/entries_direct_compare_test.go b/pkg/turbine/entries_direct_compare_test.go new file mode 100644 index 000000000..4fb55637e --- /dev/null +++ b/pkg/turbine/entries_direct_compare_test.go @@ -0,0 +1,100 @@ +package turbine + +import ( + "bytes" + "context" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestDataShredBatchMatchesChecksLengthsAndEverySlice(t *testing.T) { + raw := []byte("0123456789abcdefghijklmnopqrstuvwxyz") + shreds := []*Shred{ + {Type: ShredTypeData, Data: raw[:10]}, + nil, + {Type: ShredTypeCode, Data: []byte("coding bytes are not component data")}, + {Type: ShredTypeData}, + {Type: ShredTypeData, Data: raw[10:20]}, + {Type: ShredTypeData, Data: raw[20:]}, + } + require.True(t, dataShredBatchMatches(shreds, bytes.Clone(raw))) + for i := range raw { + changed := bytes.Clone(raw) + changed[i] ^= 1 + require.False(t, dataShredBatchMatches(shreds, changed), "changed byte %d", i) + } + for i := 0; i < len(raw); i++ { + require.False(t, dataShredBatchMatches(shreds, raw[:i]), "truncated at %d", i) + } + require.False(t, dataShredBatchMatches(shreds, append(bytes.Clone(raw), 0))) + require.True(t, dataShredBatchMatches(nil, nil)) + require.False(t, dataShredBatchMatches(nil, raw)) +} + +func TestEntryDecodeDirectComparisonCannotReuseChangedSignatureVerdict(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + txs := verifierSignedTransactions(t, 1) + raw := prefetchTestPayload(t, txs) + cache := decodedPrefetchForTest(paddedComponentShreds(t, raw, 0, 0xa5)) + cached := cache[0] + var err error + cached.verification, err = v.submitTransactions(context.Background(), entryBatchTransactions(cached.entries)) + require.NoError(t, err) + _, err = cached.verification.wait() + require.NoError(t, err) + + bad := *txs[0] + bad.Signatures = append([]solana.Signature(nil), bad.Signatures...) + bad.Signatures[0][0] ^= 1 + changed := prefetchTestPayload(t, []*solana.Transaction{&bad}) + require.Len(t, changed, len(raw)) + timings := entryDecodeTimings{prefetched: cache} + entries, _, _, err := decodeEntriesAndAlpenglowMarkersFromDataShreds(paddedComponentShreds(t, changed, 0, 0xa5), &timings) + require.NoError(t, err) + require.Len(t, timings.retained, 1) + require.NotSame(t, cached, timings.retained[0]) + require.Nil(t, timings.retained[0].verification) + blk := BlockFromEntries(100, 99, entries) + require.Equal(t, bad.Signatures[0], blk.Transactions[0].Signatures[0]) + require.ErrorContains(t, verifyDecodedEntryBatches(context.Background(), blk, timings.retained, v), "failed signature verification") + require.False(t, blk.TransactionSignaturesVerified()) +} + +func TestEntryDecodeDirectComparisonRejectsChangedCachedLength(t *testing.T) { + raw := prefetchTestPayload(t, verifierSignedTransactions(t, 1)) + for _, delta := range []int{-1, 1} { + shreds := paddedComponentShreds(t, raw, 0, 0xa5) + cache := decodedPrefetchForTest(shreds) + cached := cache[0] + if delta < 0 { + cached.raw = cached.raw[:len(cached.raw)-1] + } else { + cached.raw = append(cached.raw, 0) + } + timings := entryDecodeTimings{prefetched: cache} + entries, _, _, err := decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds, &timings) + require.NoError(t, err) + require.Len(t, entries, 1) + require.NotSame(t, cached, timings.retained[0]) + } +} + +func FuzzDataShredBatchMatchesConcatenatedBytes(f *testing.F) { + f.Add([]byte("a component spanning several shreds"), []byte("a component spanning several shreds"), uint8(3)) + f.Add([]byte("truncated"), []byte("truncate"), uint8(1)) + f.Add([]byte{}, []byte{}, uint8(0)) + f.Fuzz(func(t *testing.T, data, candidate []byte, split uint8) { + if len(data) > 64*1024 || len(candidate) > 64*1024 { + t.Skip() + } + shreds := []*Shred{nil, {Type: ShredTypeCode, Data: []byte{1, 2, 3}}} + width := int(split) + 1 + for start := 0; start < len(data); start += width { + shreds = append(shreds, &Shred{Type: ShredTypeData, Data: data[start:min(start+width, len(data))]}) + } + require.Equal(t, bytes.Equal(data, candidate), dataShredBatchMatches(shreds, candidate)) + }) +} diff --git a/pkg/turbine/entries_prefetch_test.go b/pkg/turbine/entries_prefetch_test.go new file mode 100644 index 000000000..9ecdbf599 --- /dev/null +++ b/pkg/turbine/entries_prefetch_test.go @@ -0,0 +1,146 @@ +package turbine + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func decodedPrefetchForTest(shreds []*Shred) map[uint32]*prefetchedShredBatch { + cache := make(map[uint32]*prefetchedShredBatch) + var raw []byte + var start uint32 + for _, shred := range shreds { + if raw == nil { + start = shred.Index + } + raw = append(raw, shred.Data...) + if shred.DataComplete() { + batch := decodeClosedShredBatch(raw, start, shred.Index) + batch.ready = make(chan struct{}) + close(batch.ready) + cache[start] = batch + raw = nil + } + } + return cache +} + +func TestEntryDecodeReusesOnlyExactPrefetchedBytesAndBounds(t *testing.T) { + component, err := NewEntryBatch([]Entry{{Hash: solana.Hash{9}, Txns: []solana.Transaction{mustParseTransferTx(t, 21)}}}) + require.NoError(t, err) + raw, err := MarshalBlockComponent(component) + require.NoError(t, err) + for _, mode := range []string{"match", "different_bytes", "different_end", "different_start"} { + t.Run(mode, func(t *testing.T) { + shreds := paddedComponentShreds(t, raw, 0, 0xa5) + cache := decodedPrefetchForTest(shreds) + cached := cache[0] + require.NoError(t, cached.err) + cached.parseDuration = time.Hour // must not enter final parse accounting + if mode != "match" { + cached.err = errors.New("stale cached failure must be ignored") + } + switch mode { + case "different_bytes": + cached.raw[len(cached.raw)-1] ^= 1 // even differing padding invalidates reuse + case "different_end": + cached.end++ + cached.ready = make(chan struct{}) // mismatched bounds must never wait + case "different_start": + cached.start++ + cached.ready = make(chan struct{}) + } + timings := entryDecodeTimings{prefetched: cache} + entries, _, _, err := decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds, &timings) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Len(t, timings.all, 1) + require.Len(t, timings.retained, 1) + if mode == "match" { + require.Same(t, cached, timings.retained[0]) + require.Same(t, &cached.entries[0].Txns[0], &entries[0].Txns[0]) + require.Zero(t, timings.transactionParse) + } else { + require.NotSame(t, cached, timings.retained[0]) + require.Less(t, timings.transactionParse, time.Hour) + } + }) + } +} + +func TestEntryDecodePrefetchPreparationWaitHonorsCancellation(t *testing.T) { + raw, err := marshalEntryBatch([]Entry{{Hash: solana.Hash{3}}}) + require.NoError(t, err) + shreds := paddedComponentShreds(t, raw, 0, 0) + cache := decodedPrefetchForTest(shreds) + cache[0].ready = make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + timings := entryDecodeTimings{ctx: ctx, prefetched: cache} + _, _, _, err = decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds, &timings) + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, timings.retained) + require.Zero(t, timings.transactionParse) +} + +func TestEntryDecodePrefetchPreservesUpdateParentAndIgnoresVerificationErrors(t *testing.T) { + prefix, err := NewEntryBatch([]Entry{{Hash: solana.Hash{1}, Txns: []solana.Transaction{mustParseTransferTx(t, 20)}}}) + require.NoError(t, err) + suffix, err := NewEntryBatch([]Entry{{Hash: solana.Hash{2}, Txns: []solana.Transaction{mustParseTransferTx(t, 21)}}}) + require.NoError(t, err) + components := []BlockComponent{NewBlockHeader(99, solana.Hash{9}), prefix, NewUpdateParent(98, solana.Hash{8}), suffix, NewBlockFooter(BlockFooter{BankHash: solana.Hash{7}})} + var shreds []*Shred + for _, component := range components { + raw, err := MarshalBlockComponent(component) + require.NoError(t, err) + shreds = append(shreds, paddedComponentShreds(t, raw, uint32(len(shreds)), 0xa5)...) + } + cache := decodedPrefetchForTest(shreds) + for _, start := range []uint32{32, 96} { + // Decoding must not make a signature decision, even for retained entries. + cache[start].verification = &transactionVerification{err: errors.New("signature result belongs to completion")} + cache[start].submitErr = errors.New("submission result belongs to completion") + } + timings := entryDecodeTimings{prefetched: cache} + entries, parent, footer, err := decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds, &timings) + require.NoError(t, err) + require.Len(t, entries, 1) + require.Equal(t, solana.Hash{2}, entries[0].Hash) + require.Equal(t, uint32(64), parent.ReplayFECSetIndex) + require.Equal(t, solana.Hash{7}, footer.BankHash) + require.Len(t, timings.all, 5) + require.Len(t, timings.retained, 1) + require.Same(t, cache[96], timings.retained[0]) + + // Parsing the discarded prefix remains mandatory, unlike its signatures. + cache[32].err = errors.New("malformed optimistic prefix") + timings = entryDecodeTimings{prefetched: cache} + _, _, _, err = decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds, &timings) + require.ErrorContains(t, err, "malformed optimistic prefix") +} + +func TestEntryDecodePrefetchedAgavePaddedCaptureMatchesFullDecode(t *testing.T) { + packets := agavePaddedSlot1752420Packets(t) + shreds := make([]*Shred, len(packets)) + for i, packet := range packets { + var err error + shreds[i], err = ParseShred(packet) + require.NoError(t, err) + } + wantEntries, wantParent, wantFooter, err := DecodeEntriesAndAlpenglowMarkersFromDataShreds(shreds) + require.NoError(t, err) + timings := entryDecodeTimings{prefetched: decodedPrefetchForTest(shreds)} + entries, parent, footer, err := decodeEntriesAndAlpenglowMarkersFromDataShreds(shreds, &timings) + require.NoError(t, err) + require.Equal(t, wantEntries, entries) + require.Equal(t, wantParent, parent) + require.Equal(t, wantFooter, footer) + require.Len(t, timings.all, 4) + require.Len(t, timings.retained, 2) + require.Zero(t, timings.transactionParse) +} diff --git a/pkg/turbine/entry_batch_index.go b/pkg/turbine/entry_batch_index.go new file mode 100644 index 000000000..3bf3f2640 --- /dev/null +++ b/pkg/turbine/entry_batch_index.go @@ -0,0 +1,162 @@ +package turbine + +import "math/bits" + +// Three levels cover all 65,536 permitted data-shred indexes. Successor and +// predecessor queries touch a bounded number of words, even in adversarial order. +// The index is bounded (~24 KiB per retained slot) and allocated only when +// streaming preparation is enabled. It never rescans a slot's retained map. +const batchIndexWords = maxDataShredsPerSlot / 64 + +type shredIndexBits struct { + words [batchIndexWords]uint64 + groups [batchIndexWords / 64]uint64 + top uint64 +} + +func (b *shredIndexBits) set(i uint32) { + w, g := i/64, i/4096 + b.words[w] |= uint64(1) << (i % 64) + b.groups[g] |= uint64(1) << (w % 64) + b.top |= uint64(1) << g +} +func (b *shredIndexBits) clear(i uint32) { + w, g := i/64, i/4096 + b.words[w] &^= uint64(1) << (i % 64) + if b.words[w] == 0 { + b.groups[g] &^= uint64(1) << (w % 64) + if b.groups[g] == 0 { + b.top &^= uint64(1) << g + } + } +} +func (b *shredIndexBits) next(i uint32) (uint32, bool) { + if i >= maxDataShredsPerSlot { + return 0, false + } + w, g := i/64, i/4096 + if x := b.words[w] & (^uint64(0) << (i % 64)); x != 0 { + return w*64 + uint32(bits.TrailingZeros64(x)), true + } + x := b.groups[g] & (^uint64(0) << (w%64 + 1)) + if x == 0 { + top := b.top & (^uint64(0) << (g + 1)) + if top == 0 { + return 0, false + } + g = uint32(bits.TrailingZeros64(top)) + x = b.groups[g] + } + w = g*64 + uint32(bits.TrailingZeros64(x)) + return w*64 + uint32(bits.TrailingZeros64(b.words[w])), true +} +func (b *shredIndexBits) previous(i uint32) (uint32, bool) { + if i >= maxDataShredsPerSlot { + i = maxDataShredsPerSlot - 1 + } + w, g := i/64, i/4096 + if x := b.words[w] & (^uint64(0) >> (63 - i%64)); x != 0 { + return w*64 + uint32(63-bits.LeadingZeros64(x)), true + } + x := b.groups[g] & ((uint64(1) << (w % 64)) - 1) + if x == 0 { + top := b.top & ((uint64(1) << g) - 1) + if top == 0 { + return 0, false + } + g = uint32(63 - bits.LeadingZeros64(top)) + x = b.groups[g] + } + w = g*64 + uint32(63-bits.LeadingZeros64(x)) + return w*64 + uint32(63-bits.LeadingZeros64(b.words[w])), true +} + +type entryBatchIndex struct { + missing shredIndexBits + ends shredIndexBits + emitted [batchIndexWords]uint64 +} + +func newEntryBatchIndex() *entryBatchIndex { + b := new(entryBatchIndex) + for i := range b.missing.words { + b.missing.words[i] = ^uint64(0) + } + for i := range b.missing.groups { + b.missing.groups[i] = ^uint64(0) + } + b.missing.top = (uint64(1) << len(b.missing.groups)) - 1 + return b +} + +// One insertion can complete its containing batch, and, if it provides a new +// DATA_COMPLETE boundary, the immediately following batch. All other batches +// are unchanged. A range is emitted once, only when every data index is present. +// The preceding boundary is mandatory unless the range starts at index zero. +func (b *entryBatchIndex) add(i uint32, dataComplete bool) (ready [2]shredBatchRange, n int) { + if i >= maxDataShredsPerSlot { + return ready, 0 + } + b.missing.clear(i) + if dataComplete { + b.ends.set(i) + } + if end, ok := b.ends.next(i); ok { + if r, ok := b.complete(end); ok { + ready[n] = r + n++ + } + } + if dataComplete { + if end, ok := b.ends.next(i + 1); ok { + if r, ok := b.complete(end); ok { + ready[n] = r + n++ + } + } + } + return +} +func (b *entryBatchIndex) complete(end uint32) (shredBatchRange, bool) { + if b.emitted[end/64]&(uint64(1)<<(end%64)) != 0 { + return shredBatchRange{}, false + } + start := uint32(0) + if end > 0 { + if prev, ok := b.ends.previous(end - 1); ok { + start = prev + 1 + } + } + if missing, ok := b.missing.next(start); ok && missing <= end { + return shredBatchRange{}, false + } + b.emitted[end/64] |= uint64(1) << (end % 64) + return shredBatchRange{start, end}, true +} + +func (s *slotState) discoverEntryBatch(sh *Shred) { + ready, n := s.batchIndex.add(sh.Index, sh.DataComplete()) + for _, r := range ready[:n] { + s.completeBatches = append(s.completeBatches, r) + if s.pipelineTrace != nil && !s.pipelineTrace.sealed { + s.pipelineTrace.discovered[r.start] = entryTraceNow() + } + } +} + +// Seed once if preparation is installed on an assembler with existing data. +// Normal ingress creates the index with the first accepted data shred. +func (a *SlotAssembler) notePrefetchShredLocked(s *slotState, sh *Shred) { + p := a.entryPrefetch + if p == nil || p.closed || p.ctx.Err() != nil || sh.Type != ShredTypeData { + return + } + if s.batchIndex == nil { + s.batchIndex = newEntryBatchIndex() + for _, existing := range s.shreds { + s.discoverEntryBatch(existing) + } + } else { + s.discoverEntryBatch(sh) + } +} diff --git a/pkg/turbine/entry_batch_index_test.go b/pkg/turbine/entry_batch_index_test.go new file mode 100644 index 000000000..5f812be8a --- /dev/null +++ b/pkg/turbine/entry_batch_index_test.go @@ -0,0 +1,106 @@ +package turbine + +import ( + "github.com/stretchr/testify/require" + "math/rand" + "testing" +) + +func TestShredIndexBitsBoundaries(t *testing.T) { + var b shredIndexBits + indexes := []uint32{0, 1, 63, 64, 65, 4095, 4096, 4097, 65534, 65535} + for _, i := range indexes { + b.set(i) + } + for i := uint32(0); i <= 65536; i++ { + var wantNext, wantPrev uint32 + var hasNext, hasPrev bool + for _, j := range indexes { + if j >= i && !hasNext { + wantNext, hasNext = j, true + } + if j <= i { + wantPrev, hasPrev = j, true + } + } + got, ok := b.next(i) + require.Equal(t, hasNext, ok) + if ok { + require.Equal(t, wantNext, got) + } + got, ok = b.previous(i) + require.Equal(t, hasPrev, ok) + if ok { + require.Equal(t, wantPrev, got) + } + } + for _, i := range indexes { + b.clear(i) + } + _, ok := b.next(0) + require.False(t, ok) + _, ok = b.previous(65535) + require.False(t, ok) + require.Zero(t, b.top) +} + +func TestEntryBatchIndexRandomArrivalMatchesOracle(t *testing.T) { + rng := rand.New(rand.NewSource(42)) + for trial := 0; trial < 100; trial++ { + const size = 257 + var ends, present [size]bool + for i := range ends { + ends[i] = rng.Intn(8) == 0 + } + ends[size-1] = true + index := newEntryBatchIndex() + emitted := map[shredBatchRange]bool{} + for _, i := range rng.Perm(size) { + present[i] = true + got, n := index.add(uint32(i), ends[i]) + want := map[shredBatchRange]bool{} + start, complete := 0, true + for j := 0; j < size; j++ { + complete = complete && present[j] + if ends[j] && present[j] { + r := shredBatchRange{uint32(start), uint32(j)} + if complete && !emitted[r] { + want[r] = true + } + start, complete = j+1, true + } + } + require.Len(t, want, n, "trial %d index %d", trial, i) + for _, r := range got[:n] { + require.True(t, want[r]) + require.False(t, emitted[r]) + emitted[r] = true + } + _, n = index.add(uint32(i), ends[i]) + require.Zero(t, n, "duplicate emitted") + } + } +} + +func BenchmarkEntryBatchIndex(b *testing.B) { + for _, reverse := range []bool{false, true} { + name := "ordered" + if reverse { + name = "reverse" + } + b.Run(name, func(b *testing.B) { + b.ReportAllocs() + for n := 0; n < b.N; n++ { + idx := newEntryBatchIndex() + for k := uint32(0); k < 65536; k++ { + i := k + if reverse { + i = 65535 - k + } + idx.add(i, i%64 == 63) + } + } + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/65536, "ns/shred") + }) + } +} diff --git a/pkg/turbine/entry_batch_transactions_test.go b/pkg/turbine/entry_batch_transactions_test.go new file mode 100644 index 000000000..28329ab03 --- /dev/null +++ b/pkg/turbine/entry_batch_transactions_test.go @@ -0,0 +1,37 @@ +package turbine + +import ( + "fmt" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestEntryBatchTransactionsPreservesEntryOwnershipAndOrder(t *testing.T) { + entries := []Entry{{}, {Txns: make([]solana.Transaction, 3)}, {}, {Txns: make([]solana.Transaction, 2)}} + txs := entryBatchTransactions(entries) + require.Len(t, txs, 5) + for i := 0; i < 3; i++ { + require.Same(t, &entries[1].Txns[i], txs[i]) + } + for i := 0; i < 2; i++ { + require.Same(t, &entries[3].Txns[i], txs[3+i]) + } + require.Empty(t, entryBatchTransactions(nil)) + require.Empty(t, entryBatchTransactions([]Entry{{}, {}})) +} + +var entryBatchBenchmarkSink []*solana.Transaction + +func BenchmarkEntryBatchTransactions(b *testing.B) { + for _, count := range []int{4, 49, 269, 33760} { + b.Run(fmt.Sprintf("tx_%d", count), func(b *testing.B) { + entries := []Entry{{Txns: make([]solana.Transaction, count/2)}, {}, {Txns: make([]solana.Transaction, count-count/2)}} + b.ReportAllocs() + for b.Loop() { + entryBatchBenchmarkSink = entryBatchTransactions(entries) + } + }) + } +} diff --git a/pkg/turbine/entry_hash.go b/pkg/turbine/entry_hash.go index 72af6bdb8..35ee7561c 100644 --- a/pkg/turbine/entry_hash.go +++ b/pkg/turbine/entry_hash.go @@ -90,14 +90,7 @@ func hashTransactions(txns []solana.Transaction) solana.Hash { } func hashSignatures(signatures [][]byte) solana.Hash { - if len(signatures) == 0 { - return solana.Hash{} - } - nodes := merkletree.HashNodes(signatures) - if root := nodes.GetRoot(); root != nil { - return solana.Hash(*root) - } - return solana.Hash{} + return solana.Hash(merkletree.HashRoot(signatures)) } func sha256Hash(data []byte) solana.Hash { diff --git a/pkg/turbine/entry_hash_bench_test.go b/pkg/turbine/entry_hash_bench_test.go new file mode 100644 index 000000000..e8d7d1038 --- /dev/null +++ b/pkg/turbine/entry_hash_bench_test.go @@ -0,0 +1,24 @@ +package turbine + +import ( + "encoding/binary" + "fmt" + "testing" +) + +func BenchmarkEntrySignatureRoot(b *testing.B) { + for _, count := range []int{1, 64, 311, 512, 1024} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + sigs := make([][]byte, count) + for i := range sigs { + sigs[i] = make([]byte, 64) + binary.LittleEndian.PutUint64(sigs[i], uint64(i)) + } + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _ = hashSignatures(sigs) + } + }) + } +} diff --git a/pkg/turbine/entry_identity_recovery_test.go b/pkg/turbine/entry_identity_recovery_test.go new file mode 100644 index 000000000..aab96f794 --- /dev/null +++ b/pkg/turbine/entry_identity_recovery_test.go @@ -0,0 +1,131 @@ +package turbine + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestEntryIdentityRecoveryVerifiesFinalTransactions(t *testing.T) { + for _, fault := range []string{"partial_identities", "wrong_pointer", "missing_range", "oversized_range", "nil_batch"} { + for _, invalidFinal := range []bool{false, true} { + name := fault + "/valid" + if invalidFinal { + name = fault + "/invalid" + } + t.Run(name, func(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + txs := verifierSignedTransactions(t, 3) + entries := []Entry{{Txns: []solana.Transaction{*txs[0], *txs[1], *txs[2]}}} + blk := &block.Block{Slot: 812, Transactions: entryBatchTransactions(entries)} + request, err := v.submitTransactions(context.Background(), blk.Transactions) + require.NoError(t, err) + _, err = request.wait() + require.NoError(t, err) + batches := []*prefetchedShredBatch{{entries: entries, verification: request}} + switch fault { + case "partial_identities": + request.identities = request.identities[:2] + case "wrong_pointer": + copyTx := *blk.Transactions[0] + blk.Transactions[0] = ©Tx + case "missing_range": + batches = nil + case "oversized_range": + batches[0].entries = append(batches[0].entries, Entry{Txns: []solana.Transaction{*txs[0]}}) + case "nil_batch": + batches = []*prefetchedShredBatch{nil} + } + if invalidFinal { + // The old request verified a different, valid transaction. + // A successful old verdict must not bless these final bytes. + copyTx := *blk.Transactions[0] + copyTx.Signatures = append([]solana.Signature(nil), copyTx.Signatures...) + copyTx.Signatures[0][0] ^= 1 + blk.Transactions[0] = ©Tx + } + err = verifyDecodedEntryBatches(context.Background(), blk, batches, v) + if invalidFinal { + require.ErrorContains(t, err, "failed signature verification") + return + } + require.NoError(t, err) + prepared, err := blk.PrepareTransactionMessageIdentities() + require.NoError(t, err) + for i, tx := range blk.Transactions { + want, err := txstatus.IdentityForTransaction(tx) + require.NoError(t, err) + require.Equal(t, want, prepared.Identity(i)) + } + }) + } + } +} + +func TestEntryIdentityRecoveryPreservesCancellationAndVerifierFailure(t *testing.T) { + blk := &block.Block{Slot: 813, Transactions: verifierSignedTransactions(t, 2)} + v := newTransactionVerifier(2, 16, nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, verifyDecodedEntryBatches(ctx, blk, nil, v), context.Canceled) + v.closeAndWait() + require.ErrorIs(t, verifyDecodedEntryBatches(context.Background(), blk, nil, v), errTransactionVerifierClosed) +} + +func TestEntryIdentityRecoveryJoinsCanceledReaders(t *testing.T) { + started := make(chan struct{}, 1) + release := make(chan struct{}) + v := newTransactionVerifier(1, 8, func(*solana.Transaction) error { + select { + case started <- struct{}{}: + default: + } + <-release + return nil + }) + defer v.closeAndWait() + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + txs := verifierSignedTransactions(t, 2) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + request, err := v.submitTransactions(ctx, txs) + require.NoError(t, err) + select { + case <-started: + case <-time.After(time.Second): + t.Fatal("reader never started") + } + result := make(chan error, 1) + // This deliberately inconsistent range sends us through recovery while + // the old verifier still owns the block's transaction buffers. + go func() { + result <- verifyDecodedEntryBatches(ctx, &block.Block{Transactions: txs}, []*prefetchedShredBatch{{verification: request}}, v) + }() + cancel() + select { + case err := <-result: + t.Fatalf("returned before old reader released its buffers: %v", err) + case <-time.After(20 * time.Millisecond): + } + unblock() + select { + case err := <-result: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(time.Second): + t.Fatal("recovery did not join canceled readers") + } + select { + case <-request.done: + default: + t.Fatal("request ownership was not released") + } +} diff --git a/pkg/turbine/entry_pipeline_trace.go b/pkg/turbine/entry_pipeline_trace.go new file mode 100644 index 000000000..c2c83b454 --- /dev/null +++ b/pkg/turbine/entry_pipeline_trace.go @@ -0,0 +1,212 @@ +package turbine + +// Temporary, opt-in pipeline diagnostics. No transaction bytes or keys are logged. +// Timestamps are monotonic nanoseconds relative to origin_unix_ns. Worker elapsed +// time includes descheduling; summed job durations are NOT wall-clock critical paths. +import ( + "context" + "encoding/json" + "fmt" + "os" + "strconv" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" +) + +var entryTraceOrigin = time.Now() +var entryTraceDropped atomic.Uint64 +var entryTraceConfig = configureEntryTrace() + +type entryTraceSettings struct { + modulo uint64 + until time.Time + reports chan entryPipelineReport +} + +func configureEntryTrace() entryTraceSettings { + n, err := strconv.ParseUint(os.Getenv("MITHRIL_ENTRY_TRACE_MOD"), 10, 32) + if err != nil || n == 0 { + return entryTraceSettings{} + } + seconds, err := strconv.Atoi(os.Getenv("MITHRIL_ENTRY_TRACE_SECONDS")) + if err != nil || seconds < 1 || seconds > 1800 { + seconds = 600 + } + path := os.Getenv("MITHRIL_ENTRY_TRACE_FILE") + if path == "" { + return entryTraceSettings{} + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + fmt.Fprintf(os.Stderr, "entry pipeline trace disabled: %v\n", err) + return entryTraceSettings{} + } + ch := make(chan entryPipelineReport, 8) + go func() { + defer f.Close() + enc := json.NewEncoder(f) + for r := range ch { + r.Dropped = entryTraceDropped.Load() + r.finish() + if err := enc.Encode(r); err != nil { + entryTraceDropped.Add(1) + } + } + }() + return entryTraceSettings{n, time.Now().Add(time.Duration(seconds) * time.Second), ch} +} + +func entryTraceNow() int64 { return time.Since(entryTraceOrigin).Nanoseconds() } +func entryTraceTime(t time.Time) int64 { return t.Sub(entryTraceOrigin).Nanoseconds() } + +type entryTraceContextKey struct{} + +func withEntryPipelineTrace(ctx context.Context, t *entryPipelineTrace) context.Context { + if t == nil { + return ctx + } + return context.WithValue(ctx, entryTraceContextKey{}, true) +} +func entryTraceContext(ctx context.Context) bool { + return ctx != nil && ctx.Value(entryTraceContextKey{}) == true +} + +type entryPipelineTrace struct { + arrivals map[uint32]int64 + discovered map[uint32]int64 + sealed bool // frozen at first completion claim, including retry/error paths +} + +// Called only after successful admission, under the assembler lock. Includes +// FEC-reconstructed data. This is local availability, not a NIC timestamp. +func (s *slotState) traceAcceptedShred(sh *Shred) { + if sh.Type != ShredTypeData { + return + } + if s.pipelineTrace == nil { + c := entryTraceConfig + if c.modulo == 0 || s.slot%c.modulo != 0 || time.Now().After(c.until) { + return + } + s.pipelineTrace = &entryPipelineTrace{arrivals: make(map[uint32]int64), discovered: make(map[uint32]int64)} + } + if !s.pipelineTrace.sealed { + s.pipelineTrace.arrivals[sh.Index] = entryTraceNow() + } +} + +type entryVerificationTrace struct { + Transactions int `json:"transactions"` + Submit int64 `json:"submit_ns"` + Admitted int64 `json:"admitted_ns"` + FirstWorker int64 `json:"first_worker_ns"` + LastWorker int64 `json:"last_worker_ns"` + Finished int64 `json:"finished_ns"` + Jobs int `json:"jobs"` + JobWaitSum int64 `json:"job_offer_to_start_sum_ns"` + JobWaitMax int64 `json:"job_offer_to_start_max_ns"` + WorkerSum int64 `json:"worker_elapsed_sum_ns"` + WorkerMax int64 `json:"worker_elapsed_max_ns"` +} + +func (t *entryVerificationTrace) observe(j *transactionVerifyJob) { + if t.FirstWorker == 0 || j.workerStart < t.FirstWorker { + t.FirstWorker = j.workerStart + } + t.LastWorker = max(t.LastWorker, j.workerEnd) + t.Jobs++ + wait, work := j.workerStart-j.offeredAt, j.workerEnd-j.workerStart + t.JobWaitSum += wait + t.JobWaitMax = max(t.JobWaitMax, wait) + t.WorkerSum += work + t.WorkerMax = max(t.WorkerMax, work) +} + +type entryBatchTraceReport struct { + Start uint32 `json:"start"` + End uint32 `json:"end"` + Transactions int `json:"transactions"` + Retained bool `json:"retained"` + Prefetched bool `json:"prefetched"` + AvailabilityKnown bool `json:"availability_known"` + Available int64 `json:"available_ns"` + Discovered int64 `json:"discovered_ns"` + DecodeStart int64 `json:"decode_start_ns"` + DecodeEnd int64 `json:"decode_end_ns"` + Verification *entryVerificationTrace `json:"verification,omitempty"` +} + +type entryPipelineReport struct { + Origin int64 `json:"origin_unix_ns"` + Slot uint64 `json:"slot"` + Transactions int `json:"transactions"` + Full int64 `json:"full_ns"` + CompletionStart int64 `json:"completion_start_ns"` + Ready int64 `json:"ready_ns"` + Dropped uint64 `json:"dropped_reports"` + Batches []entryBatchTraceReport `json:"batches"` + Fallback *entryVerificationTrace `json:"fallback,omitempty"` + source *entryPipelineTrace + all, retained []*prefetchedShredBatch + fallback *transactionVerification +} + +func completedEntryVerification(r *transactionVerification) *entryVerificationTrace { + if r == nil { + return nil + } + select { + case <-r.done: + return r.trace + default: + return nil + } +} + +// All source maps are sealed, and decode has joined all preparation readers. +// Reading request metrics additionally requires the verification done barrier. +func (r *entryPipelineReport) finish() { + retained := make(map[*prefetchedShredBatch]bool, len(r.retained)) + for _, b := range r.retained { + retained[b] = true + } + for _, b := range r.all { + row := entryBatchTraceReport{Start: b.start, End: b.end, Retained: retained[b], Prefetched: b.ready != nil, + DecodeStart: b.traceDecodeStart, DecodeEnd: b.traceDecodeEnd, Discovered: r.source.discovered[b.start], + AvailabilityKnown: true, Verification: completedEntryVerification(b.verification)} + for _, e := range b.entries { + row.Transactions += len(e.Txns) + } + // Require the preceding DATA_COMPLETE boundary as well as every shred in + // this batch. An end marker alone cannot establish an independent start. + start := b.start + if start > 0 { + start-- + } + for i := start; i <= b.end; i++ { + at, ok := r.source.arrivals[i] + if !ok { + row.AvailabilityKnown = false + } + row.Available = max(row.Available, at) + } + r.Batches = append(r.Batches, row) + } + r.Fallback = completedEntryVerification(r.fallback) +} + +func queueEntryPipelineReport(s *slotState, b *block.Block, d *entryDecodeTimings, start, ready time.Time) { + if s.pipelineTrace == nil || len(b.Transactions) < 10000 || entryTraceConfig.reports == nil { + return + } + r := entryPipelineReport{Origin: entryTraceOrigin.UnixNano(), Slot: s.slot, Transactions: len(b.Transactions), + Full: entryTraceTime(s.fullAt), CompletionStart: entryTraceTime(start), Ready: entryTraceTime(ready), + source: s.pipelineTrace, all: d.all, retained: d.retained, fallback: d.traceFallback} + select { + case entryTraceConfig.reports <- r: + default: + entryTraceDropped.Add(1) + } +} diff --git a/pkg/turbine/entry_pipeline_trace_test.go b/pkg/turbine/entry_pipeline_trace_test.go new file mode 100644 index 000000000..6aa493601 --- /dev/null +++ b/pkg/turbine/entry_pipeline_trace_test.go @@ -0,0 +1,68 @@ +package turbine + +import ( + "context" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestEntryPipelineTraceRequiresCompleteBatchAndBoundary(t *testing.T) { + source := &entryPipelineTrace{arrivals: map[uint32]int64{0: 10, 1: 100, 2: 20, 3: 30, 4: 40}, discovered: map[uint32]int64{0: 110, 3: 120}, sealed: true} + first := &prefetchedShredBatch{start: 0, end: 2} + later := &prefetchedShredBatch{start: 3, end: 4} + r := entryPipelineReport{source: source, all: []*prefetchedShredBatch{first, later}, retained: []*prefetchedShredBatch{later}} + r.finish() + require.Equal(t, int64(100), r.Batches[0].Available) + require.Equal(t, int64(40), r.Batches[1].Available) + require.Equal(t, int64(80), r.Batches[1].Discovered-r.Batches[1].Available, "a gap in the earlier batch delays discovery, not availability of the later one") + require.False(t, r.Batches[0].Retained) + require.True(t, r.Batches[1].Retained) + delete(source.arrivals, 2) + r.Batches = nil + r.finish() + require.False(t, r.Batches[1].AvailabilityKnown, "the preceding boundary must also have been observed") +} + +func TestEntryVerificationTraceJoinsWorkerTimings(t *testing.T) { + entered, release := make(chan struct{}), make(chan struct{}) + v := newTransactionVerifier(1, 8, func(*solana.Transaction) error { + select { + case <-entered: + default: + close(entered) + } + <-release + return nil + }) + defer v.closeAndWait() + ctx := withEntryPipelineTrace(context.Background(), &entryPipelineTrace{}) + r, err := v.submitTransactions(ctx, []*solana.Transaction{{}}) + require.NoError(t, err) + <-entered + require.Nil(t, completedEntryVerification(r), "unfinished mutable metrics must not be read") + close(release) + _, err = r.wait() + require.NoError(t, err) + m := completedEntryVerification(r) + require.NotNil(t, m) + require.Equal(t, 1, m.Jobs) + require.LessOrEqual(t, m.Submit, m.Admitted) + require.LessOrEqual(t, m.Admitted, m.FirstWorker) + require.Less(t, m.FirstWorker, m.LastWorker) + require.LessOrEqual(t, m.LastWorker, m.Finished) + require.Positive(t, m.WorkerSum) + require.GreaterOrEqual(t, m.JobWaitSum, int64(0)) + plain, err := v.submitTransactions(context.Background(), []*solana.Transaction{{}}) + require.NoError(t, err) + _, err = plain.wait() + require.NoError(t, err) + require.Nil(t, plain.trace, "ordinary verification does not collect job timestamps") +} + +func TestEntryPipelineTraceSealedGeneration(t *testing.T) { + s := &slotState{pipelineTrace: &entryPipelineTrace{arrivals: map[uint32]int64{1: 42}, discovered: map[uint32]int64{}, sealed: true}} + s.traceAcceptedShred(&Shred{Type: ShredTypeData, Index: 2}) + require.Len(t, s.pipelineTrace.arrivals, 1, "completion/retry cannot mutate a report's frozen arrival map") +} diff --git a/pkg/turbine/entry_prefetch.go b/pkg/turbine/entry_prefetch.go new file mode 100644 index 000000000..2b082e52b --- /dev/null +++ b/pkg/turbine/entry_prefetch.go @@ -0,0 +1,428 @@ +package turbine + +import ( + "context" + "errors" + "sync" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" +) + +const ( + entryPrefetchSlots = 8 + // Covers a large block of maximum-size transactions while bounding the + // extra retained encoded bytes across all in-flight generations. + entryPrefetchBytes = 64 << 20 + entryPrefetchBatchBytes = 1 << 20 +) + +type shredBatchRange struct{ start, end uint32 } + +// A result belongs to one exact DATA_COMPLETE range in one slot generation. +// Fields are immutable after ready closes; signature readers own its decoded +// transactions until verification.done closes. +type prefetchedShredBatch struct { + start, end uint32 + raw []byte + entries []Entry + parent *AlpenglowParentInfo + footer *BlockFooter + marker bool + traceDecodeStart int64 + traceDecodeEnd int64 + parseDuration time.Duration + err error + ready chan struct{} + verification *transactionVerification + submittedAt time.Time + submitErr error +} + +type slotEntryPrefetch struct { + pool *entryPrefetchPool + ctx context.Context + cancel context.CancelFunc + batches map[uint32]*prefetchedShredBatch + next int + queued, released bool + queueDone chan struct{} // closed after the queued/running token retires + bytes int +} + +// All scheduling and accounting use assembler.mu. The packet reader only +// indexes complete data ranges and attempts a nonblocking, coalesced enqueue. +// Decoding and bounded verifier admission run on separate background workers. +type entryPrefetchPool struct { + a *SlotAssembler + ctx context.Context + cancel context.CancelFunc + verifier *transactionVerifier + jobs chan *slotState + workers, cleanup sync.WaitGroup + slots, bytes int + closed bool + close sync.Once +} + +func newEntryPrefetchPool(ctx context.Context, a *SlotAssembler, verifier *transactionVerifier) *entryPrefetchPool { + ctx, cancel := context.WithCancel(ctx) + p := &entryPrefetchPool{a: a, ctx: ctx, cancel: cancel, verifier: verifier, jobs: make(chan *slotState, entryPrefetchSlots)} + a.mu.Lock() + a.entryPrefetch = p + a.mu.Unlock() + p.workers.Add(2) + for i := 0; i < 2; i++ { + go p.run() + } + return p +} + +func (a *SlotAssembler) prefetchEntriesLocked(s *slotState) { + p := a.entryPrefetch + if p == nil || p.closed || p.ctx.Err() != nil { + return + } + if s.batchIndex == nil && len(s.shreds) != 0 { + s.batchIndex = newEntryBatchIndex() + for _, sh := range s.shreds { + s.discoverEntryBatch(sh) + } + } + if s.prefetch == nil && len(s.completeBatches) > 0 && p.slots < entryPrefetchSlots { + ctx, cancel := context.WithCancel(withEntryPipelineTrace(p.ctx, s.pipelineTrace)) + s.prefetch = &slotEntryPrefetch{pool: p, ctx: ctx, cancel: cancel, batches: make(map[uint32]*prefetchedShredBatch)} + p.slots++ + } + p.enqueueLocked(s) +} + +func (p *entryPrefetchPool) enqueueLocked(s *slotState) { + f := s.prefetch + if f == nil || f.pool != p || f.queued || f.released || s.completing || p.closed || f.next >= len(s.completeBatches) { + return + } + select { + case p.jobs <- s: + f.queued = true + f.queueDone = make(chan struct{}) + default: + } +} + +func (p *entryPrefetchPool) run() { + defer p.workers.Done() + for s := range p.jobs { + p.a.mu.Lock() + f := s.prefetch + if p.closed || f.released || f.ctx.Err() != nil || p.a.slots[s.slot] != s || s.completing { + f.queued = false + close(f.queueDone) + p.a.mu.Unlock() + continue + } + var batch *prefetchedShredBatch + var shreds []*Shred + var rawSize int + for f.next < len(s.completeBatches) { + r := s.completeBatches[f.next] + size := 0 + for i := r.start; i <= r.end; i++ { + size += len(s.shreds[i].Data) + if size > entryPrefetchBatchBytes { + break + } + } + if size > entryPrefetchBatchBytes { + f.next++ + continue + } + if p.bytes+size > entryPrefetchBytes { + break + } + f.next++ + p.bytes += size + f.bytes += size + rawSize = size + batch = &prefetchedShredBatch{start: r.start, end: r.end, ready: make(chan struct{})} + f.batches[r.start] = batch + shreds = make([]*Shred, 0, r.end-r.start+1) + for i := r.start; i <= r.end; i++ { + shreds = append(shreds, s.shreds[i]) + } + break + } + if batch == nil { + f.queued = false + close(f.queueDone) + p.a.mu.Unlock() + continue + } + p.a.mu.Unlock() + + if s.pipelineTrace != nil { + batch.traceDecodeStart = entryTraceNow() + } + raw := make([]byte, 0, rawSize) + for _, sh := range shreds { + raw = append(raw, sh.Data...) + } + decoded := decodeClosedShredBatch(raw, batch.start, batch.end) + ready := batch.ready + batch.raw, batch.entries = decoded.raw, decoded.entries + batch.parent, batch.footer, batch.marker = decoded.parent, decoded.footer, decoded.marker + batch.parseDuration, batch.err = decoded.parseDuration, decoded.err + if s.pipelineTrace != nil { + batch.traceDecodeEnd = entryTraceNow() + } + if batch.err == nil && !batch.marker && f.ctx.Err() == nil { + txs := entryBatchTransactions(batch.entries) + if len(txs) > 0 { + batch.submittedAt = time.Now() + batch.verification, batch.submitErr = p.verifier.submitPrefetchTransactions(f.ctx, txs) + } + } + close(ready) + p.a.mu.Lock() + f.queued = false + close(f.queueDone) + p.enqueueLocked(s) + p.a.mu.Unlock() + } +} + +func entryBatchTransactions(entries []Entry) []*solana.Transaction { + count := 0 + for i := range entries { + count += len(entries[i].Txns) + } + // Entries are already decoded: size this pointer view once instead of + // repeatedly reallocating and copying it while preparing each component. + txs := make([]*solana.Transaction, 0, count) + for i := range entries { + for j := range entries[i].Txns { + txs = append(txs, &entries[i].Txns[j]) + } + } + return txs +} + +// Keep reservations until canceled readers have actually relinquished their +// buffers. Repeated resets cannot evade the memory or slot bounds. +func (a *SlotAssembler) releasePrefetchLocked(s *slotState) { + if s == nil || s.prefetch == nil || s.prefetch.released { + return + } + f := s.prefetch + f.released = true + f.cancel() + p := f.pool + queueDone := f.queueDone + p.cleanup.Add(1) + go func() { + defer p.cleanup.Done() + for _, b := range f.batches { + <-b.ready + if b.verification != nil { + b.verification.wait() + } + } + if queueDone != nil { + <-queueDone + } + p.a.mu.Lock() + p.slots-- + p.bytes -= f.bytes + p.a.mu.Unlock() + }() +} + +func (p *entryPrefetchPool) closeAndWait() { + p.close.Do(func() { + p.cancel() + p.a.mu.Lock() + p.closed = true + if p.a.entryPrefetch == p { + p.a.entryPrefetch = nil + } + for _, s := range p.a.slots { + if s.prefetch != nil && s.prefetch.pool == p { + p.a.releasePrefetchLocked(s) + } + } + close(p.jobs) + p.a.mu.Unlock() + p.workers.Wait() + p.cleanup.Wait() + }) +} + +// Reuse only retained entry results. UpdateParent may intentionally discard an +// invalid optimistic prefix. Unprefetched transactions form one immediately +// available request, overlapping any early requests still running. +func verifyDecodedEntryBatches(ctx context.Context, blk *block.Block, batches []*prefetchedShredBatch, verifier *transactionVerifier) error { + return verifyDecodedEntryBatchesWithTimings(ctx, blk, batches, verifier, nil) +} + +func verifyDecodedEntryBatchesWithTimings(ctx context.Context, blk *block.Block, batches []*prefetchedShredBatch, verifier *transactionVerifier, timings *entryDecodeTimings) error { + if ctx == nil { + ctx = context.Background() + } + if blk == nil { + return errors.New("verify decoded entries: nil block") + } + type pending struct { + future *transactionVerification + offset int + count int + } + var early []pending + var missing []*solana.Transaction + var indices []int + offset := 0 + for _, b := range batches { + if b == nil { + return recoverEntryVerification(ctx, blk, batches, verifier, errors.New("nil retained entry batch")) + } + count := 0 + for _, e := range b.entries { + count += len(e.Txns) + } + if count > len(blk.Transactions)-offset { + return recoverEntryVerification(ctx, blk, batches, verifier, errors.New("entry transaction range exceeds final block")) + } + reusable := b.verification != nil + if reusable { + select { + case <-b.verification.done: + // Cancellation is not a signature verdict. A completion canceled + // after preparation may be retried on this same slot generation. + reusable = !errors.Is(b.verification.err, context.Canceled) && !errors.Is(b.verification.err, context.DeadlineExceeded) + default: + } + } + if reusable { + early = append(early, pending{b.verification, offset, count}) + } else { + missing = append(missing, blk.Transactions[offset:offset+count]...) + for i := 0; i < count; i++ { + indices = append(indices, offset+i) + } + } + offset += count + } + if offset != len(blk.Transactions) { + return recoverEntryVerification(ctx, blk, batches, verifier, errors.New("entry identity coverage mismatch")) + } + var fallback *transactionVerification + var err error + if len(missing) > 0 { + fallback, err = verifier.submitTransactions(ctx, missing) + if timings != nil { + timings.traceFallback = fallback + } + } + firstIndex := len(blk.Transactions) + firstErr := err + for _, p := range early { + i, e := p.future.waitContext(ctx) + if e != nil && (firstErr == nil || (i >= 0 && p.offset+i < firstIndex)) { + firstErr = e + if i >= 0 { + firstIndex = p.offset + i + } + } + } + if fallback != nil { + i, e := fallback.waitContext(ctx) + if e != nil && (firstErr == nil || (i >= 0 && indices[i] < firstIndex)) { + firstErr = e + if i >= 0 { + firstIndex = indices[i] + } + } + } + if ctx.Err() != nil { + return ctx.Err() + } + if firstErr != nil && firstIndex < len(blk.Transactions) { + return formatTransactionVerificationError(blk, firstIndex, firstErr) + } + if firstErr != nil { + return firstErr + } + // Custom verification hooks do not produce trusted message identities. + // Preserve their existing lazy preparation path (primarily test fixtures). + if verifier.verify != nil { + return nil + } + identities := make([]txverify.VerifiedMessageIdentity, len(blk.Transactions)) + for _, p := range early { + if len(p.future.identities) != p.count || p.offset+len(p.future.identities) > len(identities) { + return recoverEntryVerification(ctx, blk, batches, verifier, errors.New("entry identity range mismatch")) + } + copy(identities[p.offset:], p.future.identities) + } + if fallback != nil { + if len(fallback.identities) != len(indices) { + return recoverEntryVerification(ctx, blk, batches, verifier, errors.New("fallback identity coverage mismatch")) + } + for i, index := range indices { + identities[index] = fallback.identities[i] + } + } + if err := blk.CacheVerifiedTransactionMessageIdentities(identities); err != nil { + return recoverEntryVerification(ctx, blk, batches, verifier, err) + } + return nil +} + +// Prefetch metadata is an optimization, never a substitute for verifying the +// final block. Join old readers, then verify every final transaction afresh. +// This also repairs pointer/coverage mismatches without treating a successful +// verdict for another transaction as proof for this one. Normal signature +// failures above are still rejected directly. Failed recovery stays an error. +func recoverEntryVerification(ctx context.Context, blk *block.Block, batches []*prefetchedShredBatch, verifier *transactionVerifier, reason error) error { + for _, batch := range batches { + if batch != nil && batch.verification != nil { + _, _ = batch.verification.waitContext(ctx) + } + } + if err := ctx.Err(); err != nil { + return err + } + mlog.Log.Warnf("slot %d: discarded inconsistent entry verification metadata; re-verifying final transactions: %v", blk.Slot, reason) + return verifier.verifyBlockContext(ctx, blk) +} + +func earlyEntryTimings(t *entryDecodeTimings, fullAt time.Time, timings *block.TurbineIngressTimings) { + for _, b := range t.all { + if b.ready == nil { + continue + } + timings.EarlyTransactionParse += b.parseDuration + if b.verification != nil { + select { + case <-b.verification.done: + timings.EarlyTransactionSigverify += b.verification.finishedAt.Sub(b.submittedAt) + default: + } + } + } + for _, b := range t.retained { + if b.verification != nil { + select { + case <-b.verification.done: + if b.verification.err == nil && !b.verification.finishedAt.After(fullAt) { + for _, e := range b.entries { + timings.EarlyVerifiedTransactions += uint64(len(e.Txns)) + } + } + default: + } + } + } +} diff --git a/pkg/turbine/entry_prefetch_benchmark_test.go b/pkg/turbine/entry_prefetch_benchmark_test.go new file mode 100644 index 000000000..be1ef6dcd --- /dev/null +++ b/pkg/turbine/entry_prefetch_benchmark_test.go @@ -0,0 +1,288 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "encoding/binary" + "fmt" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/sigverify" + "github.com/gagliardetto/solana-go" +) + +// BenchmarkEntryPrefetchAssembly runs actual assembler ingestion, component +// decoding, final Merkle identity checks, transaction verification, and final +// publication. Source transactions use the same generated 228/1232-byte or +// captured fixtures as BenchmarkTransactionVerificationFlow. Generation, +// packet parsing and shred-signature authentication happen outside the timer. +// There is no network I/O, loss/recovery, replay, PoH or footer-bankhash execution. +// +// Complete component bursts are scheduled across 200 ms for tip scenarios; +// catchup offers all shreds immediately. This is a workload model, not a +// measured cluster arrival distribution. A 60 KiB signed-transaction budget +// defines components; actual entry overhead and FEC padding are generated. +// Pools persist across iterations; the identical fixture slot is reset between +// samples outside the timer. Use fixed counts (e.g. -benchtime=3x). +func BenchmarkEntryPrefetchAssembly(b *testing.B) { + flowConfigureBackend(b) + for _, source := range flowBenchmarkFixtures(b) { + b.Run(source.name, func(b *testing.B) { + fixture := makeAssemblyFlowFixture(b, source.blk) + for _, workers := range []int{2, 4} { + b.Run(fmt.Sprintf("workers_%d", workers), func(b *testing.B) { + for _, target := range []int{4, 8} { + b.Run(fmt.Sprintf("target_%d", target), func(b *testing.B) { + for _, arrival := range []struct { + name string + span time.Duration + }{{"catchup", 0}, {"tip_200ms", 200 * time.Millisecond}} { + b.Run(arrival.name, func(b *testing.B) { + for _, overlap := range []bool{false, true} { + name := "overlap_off" + if overlap { + name = "overlap_on" + } + b.Run(name, func(b *testing.B) { + runAssemblyFlowBenchmark(b, source.blk, fixture, workers, target, arrival.span, overlap, false) + }) + } + }) + } + }) + } + }) + } + }) + } +} + +type assemblyFlowFixture struct { + components [][]*Shred + authenticatedRoots [][]solana.Hash + blockID solana.Hash + parentID solana.Hash + bankhash solana.Hash + shreds int + holdGap bool +} + +func makeAssemblyFlowFixture(tb testing.TB, source *block.Block) assemblyFlowFixture { + tb.Helper() + fixture := assemblyFlowFixture{parentID: solana.Hash{31}, bankhash: solana.Hash{47}} + var seed [ed25519.SeedSize]byte + seed[0] = 193 // Public deterministic benchmark leader, not validator material. + leader := solana.PrivateKey(ed25519.NewKeyFromSeed(seed[:])) + public := solana.PublicKeyFromBytes(ed25519.PrivateKey(leader).Public().(ed25519.PublicKey)) + generator := ShredGenerator{Slot: 100, ParentSlot: 99, Version: 7, ReferenceTick: 63} + var nextData, nextCode uint32 + var chained solana.Hash + var roots []solana.Hash + appendComponent := func(component BlockComponent, last bool) { + raw, err := MarshalBlockComponent(component) + if err != nil { + tb.Fatal(err) + } + packets, finalRoot, dataEnd, codeEnd, err := generator.MakeShredsFromData(leader, raw, last, chained, nextData, nextCode) + if err != nil { + tb.Fatal(err) + } + chained, nextData, nextCode = finalRoot, dataEnd, codeEnd + var data []*Shred + var authenticatedRoots []solana.Hash + for _, packet := range packets { + shred, err := ParseShred(packet) + if err != nil { + tb.Fatal(err) + } + if shred.Type != ShredTypeData { + continue + } + if err := shred.VerifySignature(public); err != nil { + tb.Fatal(err) + } + root, err := shred.MerkleRoot() + if err != nil { + tb.Fatal(err) + } + if shred.Index == shred.FECSetIndex { + roots = append(roots, root) + } + authenticatedRoots = append(authenticatedRoots, root) + data = append(data, shred) + } + fixture.components = append(fixture.components, data) + fixture.authenticatedRoots = append(fixture.authenticatedRoots, authenticatedRoots) + fixture.shreds += len(data) + } + appendComponent(NewBlockHeader(99, fixture.parentID), false) + for i, txs := range flowComponents(source.Transactions, 60*1024) { + entry := Entry{NumHashes: 1, Txns: make([]solana.Transaction, len(txs))} + binary.LittleEndian.PutUint64(entry.Hash[:], uint64(i+1)) + for j, tx := range txs { + entry.Txns[j] = *tx + } + appendComponent(BlockComponent{EntryBatch: []Entry{entry}}, false) + } + appendComponent(NewBlockFooter(BlockFooter{BankHash: fixture.bankhash}), true) + if nextData > maxDataShredsPerSlot { + tb.Fatalf("fixture requires %d data shreds; assembler limit is %d", nextData, maxDataShredsPerSlot) + } + fixture.blockID = DoubleMerkleBlockID(99, fixture.parentID, roots) + return fixture +} + +func runAssemblyFlowBenchmark(b *testing.B, source *block.Block, fixture assemblyFlowFixture, workers, target int, span time.Duration, overlap, prepareIdentities bool) { + v := newTransactionVerifierWithBatchTarget(workers, 2*workers*target, target, nil) + defer v.closeAndWait() + if err := v.verifyBlock(source); err != nil { + b.Fatal(err) + } + a := NewSlotAssembler() + a.verifyTransactions = v.verifyBlockContext + a.SetKnownAlpenglowBlockID(99, fixture.parentID) + a.SetKnownAlpenglowBlockID(100, fixture.blockID) + if overlap { + prefetch := newEntryPrefetchPool(context.Background(), a, v) + defer prefetch.closeAndWait() + } + var ready, parse, preparation, joins, arrivals, identityPreparation, fullToIdentities []time.Duration + var early uint64 + var cpu float64 + before := sigverify.Stats() + b.ReportAllocs() + b.ResetTimer() + for range b.N { + b.StopTimer() + a.ResetSlot(100) + cpuStarted := flowCPUSeconds(b) + b.StartTimer() + started := time.Now() + var work *slotCompletionWork + for componentIndex, component := range fixture.components { + if span > 0 { + time.Sleep(time.Until(started.Add(flowArrivalOffset(componentIndex, len(fixture.components), span)))) + } + for shredIndex, shred := range component { + if fixture.holdGap && componentIndex == len(fixture.components)*3/4 && shredIndex == 1 { + continue + } + candidate, err := a.addShredFromWithRoot(shred, false, &fixture.authenticatedRoots[componentIndex][shredIndex]) + if err != nil { + b.Fatal(err) + } + if candidate != nil { + if work != nil { + b.Fatal("slot claimed completion more than once") + } + work = candidate + } + } + } + if fixture.holdGap { + ci := len(fixture.components) * 3 / 4 + candidate, err := a.addShredFromWithRoot(fixture.components[ci][1], false, &fixture.authenticatedRoots[ci][1]) + if err != nil || candidate == nil || work != nil { + b.Fatalf("held gap completion failed: %v", err) + } + work = candidate + } + if work == nil { + b.Fatal("all generated data shreds did not complete the slot") + } + processed := a.processCompletion(context.Background(), work) + completed, err := a.finalizeCompletion(work, processed) + if prepareIdentities && err == nil && completed != nil { + start := time.Now() + _, err = completed.PrepareTransactionMessageIdentities() + identityPreparation = append(identityPreparation, time.Since(start)) + fullToIdentities = append(fullToIdentities, time.Since(work.state.fullAt)) + } + b.StopTimer() + cpu += flowCPUSeconds(b) - cpuStarted + if err != nil || completed == nil { + b.Fatalf("completion failed: block=%v error=%v", completed != nil, err) + } + if !completed.TransactionSignaturesVerified() || len(completed.Transactions) != len(source.Transactions) || + !completed.HasAlpenglowBlockID || solana.Hash(completed.AlpenglowBlockID) != fixture.blockID || + !completed.HasExpectedBankhash || completed.ExpectedBankhash != fixture.bankhash { + b.Fatal("completion changed transaction coverage or authenticated block metadata") + } + ready = append(ready, processed.timings.FullToReady) + parse = append(parse, processed.timings.TransactionParse) + preparation = append(preparation, processed.timings.EarlyPreparationWait) + joins = append(joins, processed.timings.TransactionSigverify) + arrivals = append(arrivals, processed.timings.ShredCollection) + early += processed.timings.EarlyVerifiedTransactions + } + after := sigverify.Stats() + b.ReportMetric(cpu*1000/float64(b.N), "cpu-ms/block") + b.ReportMetric(cpu/b.Elapsed().Seconds(), "avg_cpu_cores") + b.ReportMetric(float64(early)/float64(b.N), "early_verified_tx/block") + b.ReportMetric(float64(len(source.Transactions)), "tx/block") + b.ReportMetric(float64(fixture.shreds), "data_shreds/block") + b.ReportMetric(float64(len(fixture.components)), "components/block") + if batches := after.Batches - before.Batches; batches > 0 { + b.ReportMetric(float64(after.Signatures-before.Signatures)/float64(batches), "mean_width") + } + flowReportPercentiles(b, ready, "full_to_ready") + flowReportPercentiles(b, parse, "completion_parse") + flowReportPercentiles(b, preparation, "preparation_wait") + flowReportPercentiles(b, joins, "completion_sigverify") + flowReportPercentiles(b, arrivals, "collection") + if prepareIdentities { + flowReportPercentiles(b, identityPreparation, "identity_admission") + flowReportPercentiles(b, fullToIdentities, "full_to_identities") + } + if after.InternalFaultFallbacks != before.InternalFaultFallbacks { + b.Fatal("signature verifier used an internal fault fallback") + } + var signatures uint64 + for _, tx := range source.Transactions { + signatures += uint64(len(tx.Signatures)) + } + if got, want := after.Signatures-before.Signatures, signatures*uint64(b.N); got != want { + b.Fatalf("verified %d signatures; want %d, exactly once per retained transaction", got, want) + } +} + +// BenchmarkEntryMessageIdentityArrival includes the first admission-time message +// identity lookup after assembly. Compare unchanged baseline and candidate with +// identical fixtures; full_to_identities includes any moved completion work. +// This does not include replay's whole-block duplicate map or transaction loop. +func BenchmarkEntryMessageIdentityArrival(b *testing.B) { + flowConfigureBackend(b) + for _, source := range flowBenchmarkFixtures(b) { + b.Run(source.name, func(b *testing.B) { + fixture := makeAssemblyFlowFixture(b, source.blk) + for _, arrival := range []struct { + name string + span time.Duration + }{{"catchup", 0}, {"tip_200ms", 200 * time.Millisecond}} { + b.Run(arrival.name, func(b *testing.B) { + runAssemblyFlowBenchmark(b, source.blk, fixture, 2, 8, arrival.span, true, true) + }) + } + }) + } +} + +// Holds one data shred in a batch three quarters through the block until the +// footer has arrived. Other complete batches continue arriving over 200 ms. +// This isolates discovery behind a gap; it is not a measured network replay. +func BenchmarkEntryPrefetchGapArrival(b *testing.B) { + flowConfigureBackend(b) + for _, source := range flowBenchmarkFixtures(b) { + b.Run(source.name, func(b *testing.B) { + fixture := makeAssemblyFlowFixture(b, source.blk) + for _, gap := range []bool{false, true} { + b.Run(fmt.Sprintf("gap_%t", gap), func(b *testing.B) { + fixture.holdGap = gap + runAssemblyFlowBenchmark(b, source.blk, fixture, 2, 8, 200*time.Millisecond, true, true) + }) + } + }) + } +} diff --git a/pkg/turbine/entry_prefetch_bounds_test.go b/pkg/turbine/entry_prefetch_bounds_test.go new file mode 100644 index 000000000..189ff25c3 --- /dev/null +++ b/pkg/turbine/entry_prefetch_bounds_test.go @@ -0,0 +1,50 @@ +package turbine + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestEntryPrefetchByteBoundsFallBackToCompleteVerification(t *testing.T) { + for _, mode := range []string{"budget_full", "oversized_component"} { + t.Run(mode, func(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + if mode == "budget_full" { + // Model other generations owning the entire encoded-byte budget. + a.mu.Lock() + p.bytes = entryPrefetchBytes + a.mu.Unlock() + defer func() { a.mu.Lock(); p.bytes -= entryPrefetchBytes; a.mu.Unlock() }() + } + raw := prefetchTestPayload(t, verifierSignedTransactions(t, 3)) + if mode == "oversized_component" { + // Ordinary entry components permit trailing FEC padding. The + // complete decoder must accept this even though prefetch skips it. + raw = append(raw, make([]byte, entryPrefetchBatchBytes+1-len(raw))...) + } + const slot = 400 + batches := prefetchTestShreds(t, slot, raw, buildAlpenglowEndingTick(t)) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + require.Eventually(t, func() bool { + a.mu.Lock() + defer a.mu.Unlock() + f := a.slots[slot].prefetch + return f != nil && !f.queued && len(f.batches) == 0 + }, 3*time.Second, time.Millisecond) + blk := feedPrefetchShreds(t, a, batches[1]) + require.NotNil(t, blk) + require.Len(t, blk.Transactions, 3) + require.True(t, blk.TransactionSignaturesVerified()) + timings, ok := blk.TurbineIngressTimings() + require.True(t, ok) + require.Zero(t, timings.EarlyVerifiedTransactions) + }) + } +} diff --git a/pkg/turbine/entry_prefetch_test.go b/pkg/turbine/entry_prefetch_test.go new file mode 100644 index 000000000..679ee18b5 --- /dev/null +++ b/pkg/turbine/entry_prefetch_test.go @@ -0,0 +1,528 @@ +package turbine + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func prefetchTestPayload(t *testing.T, txs []*solana.Transaction) []byte { + t.Helper() + entry := Entry{NumHashes: 1, Hash: solana.Hash{7}, Txns: make([]solana.Transaction, len(txs))} + for i, tx := range txs { + entry.Txns[i] = *tx + } + raw, err := marshalEntryBatch([]Entry{entry}) + require.NoError(t, err) + return raw +} + +func prefetchTestShreds(t *testing.T, slot uint64, payloads ...[]byte) [][]*Shred { + t.Helper() + gen := ShredGenerator{Slot: slot, ParentSlot: slot - 1, Version: 1} + var nextData, nextCode uint32 + var root solana.Hash + batches := make([][]*Shred, len(payloads)) + for i, raw := range payloads { + packets, nextRoot, data, code, err := gen.MakeShredsFromData(testShredLeader(t), raw, i == len(payloads)-1, root, nextData, nextCode) + require.NoError(t, err) + root, nextData, nextCode = nextRoot, data, code + for _, packet := range packets { + shred, err := ParseShred(packet) + require.NoError(t, err) + if shred.Type == ShredTypeData { + batches[i] = append(batches[i], shred) + } + } + } + return batches +} + +func feedPrefetchShreds(t *testing.T, a *SlotAssembler, shreds []*Shred) *block.Block { + t.Helper() + var result *block.Block + for _, shred := range shreds { + blk, err := a.AddShred(shred) + require.NoError(t, err) + if blk != nil { + require.Nil(t, result) + result = blk + } + } + return result +} + +func waitPrefetchedBatch(t *testing.T, a *SlotAssembler, slot uint64, start uint32) *prefetchedShredBatch { + t.Helper() + var batch *prefetchedShredBatch + require.Eventually(t, func() bool { + a.mu.Lock() + defer a.mu.Unlock() + if s := a.slots[slot]; s != nil && s.prefetch != nil { + batch = s.prefetch.batches[start] + } + return batch != nil + }, 3*time.Second, time.Millisecond) + waitSignal(t, batch.ready, "prefetched component preparation") + require.NoError(t, batch.err) + require.NoError(t, batch.submitErr) + return batch +} + +func TestEntryPrefetchVerifiesBeforeLastShredAndReusesResults(t *testing.T) { + var calls atomic.Int32 + v := newTransactionVerifier(2, 16, func(tx *solana.Transaction) error { + calls.Add(1) + return txverify.VerifyTransaction(tx) + }) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + const slot = 100 + batches := prefetchTestShreds(t, slot, + prefetchTestPayload(t, verifierSignedTransactions(t, 3)), + prefetchTestPayload(t, verifierSignedTransactions(t, 4))) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + cached := waitPrefetchedBatch(t, a, slot, 0) + require.NotNil(t, cached.verification) + _, err := cached.verification.wait() + require.NoError(t, err) + require.Equal(t, int32(3), calls.Load()) + for range 5 { + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + } + blk := feedPrefetchShreds(t, a, batches[1]) + require.NotNil(t, blk) + require.Len(t, blk.Transactions, 7) + require.True(t, blk.TransactionSignaturesVerified()) + require.Equal(t, int32(7), calls.Load(), "cached transactions must not be verified twice") + timings, ok := blk.TurbineIngressTimings() + require.True(t, ok) + require.Equal(t, uint64(3), timings.EarlyVerifiedTransactions) + require.LessOrEqual(t, cached.verification.finishedAt.UnixNano(), blk.ShredFullNanos) +} + +func TestEntryPrefetchWaitsForGapAcrossMultipleFECSets(t *testing.T) { + var calls atomic.Int32 + v := newTransactionVerifier(2, 16, func(*solana.Transaction) error { calls.Add(1); return nil }) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + const slot = 104 + batches := prefetchTestShreds(t, slot, + prefetchTestPayload(t, verifierSignedTransactions(t, 300)), buildAlpenglowEndingTick(t)) + require.Greater(t, len(batches[0]), dataShredsPerFECBlock) + // Arrival of DATA_COMPLETE and later FEC sets cannot bypass the hole. + for i := len(batches[0]) - 1; i >= 0; i-- { + if i != 1 { + require.Nil(t, feedPrefetchShreds(t, a, batches[0][i:i+1])) + } + } + a.mu.Lock() + require.Empty(t, a.slots[slot].completeBatches) + require.Nil(t, a.slots[slot].prefetch) + a.mu.Unlock() + require.Zero(t, calls.Load()) + require.Nil(t, feedPrefetchShreds(t, a, batches[0][1:2])) + cached := waitPrefetchedBatch(t, a, slot, 0) + _, err := cached.verification.wait() + require.NoError(t, err) + require.Equal(t, int32(300), calls.Load()) + blk := feedPrefetchShreds(t, a, batches[1]) + require.NotNil(t, blk) + require.Equal(t, int32(300), calls.Load()) +} + +func TestEntryPrefetchResetKeepsOldReservationsUntilReadersJoin(t *testing.T) { + txs := verifierSignedTransactions(t, 2) + txs[0].Signatures[0][9] ^= 0x40 + oldSignature := txs[0].Signatures[0] + started := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + v := newTransactionVerifier(1, 8, func(tx *solana.Transaction) error { + if tx.Signatures[0] == oldSignature { + close(started) + <-release + } + return txverify.VerifyTransaction(tx) + }) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + const slot = 108 + old := prefetchTestShreds(t, slot, prefetchTestPayload(t, txs[:1]), buildAlpenglowEndingTick(t)) + fresh := prefetchTestShreds(t, slot, prefetchTestPayload(t, txs[1:]), buildAlpenglowEndingTick(t)) + require.Nil(t, feedPrefetchShreds(t, a, old[0])) + waitSignal(t, started, "old generation verifier") + oldBatch := waitPrefetchedBatch(t, a, slot, 0) + a.ResetSlot(slot) + a.mu.Lock() + require.Equal(t, 1, p.slots) + require.Positive(t, p.bytes) + a.mu.Unlock() + require.Nil(t, feedPrefetchShreds(t, a, fresh[0])) + a.mu.Lock() + require.Equal(t, 2, p.slots) + a.mu.Unlock() + // With one verifier worker only one request may prefetch. The fresh + // generation keeps its pool reservation while admission waits for the old + // reader to join; it must not release or reuse the old generation's bytes. + releaseOnce.Do(func() { close(release) }) + _, err := oldBatch.verification.wait() + require.ErrorIs(t, err, context.Canceled) + newBatch := waitPrefetchedBatch(t, a, slot, 0) + require.NotSame(t, oldBatch, newBatch) + _, err = newBatch.verification.wait() + require.NoError(t, err) + blk := feedPrefetchShreds(t, a, fresh[1]) + require.NotNil(t, blk) + require.Equal(t, txs[1].Signatures[0], blk.Transactions[0].Signatures[0]) + require.Eventually(t, func() bool { + a.mu.Lock() + defer a.mu.Unlock() + return p.slots == 0 && p.bytes == 0 + }, 3*time.Second, time.Millisecond) +} + +func TestEntryPrefetchSaturationDoesNotBlockShredAdmissionAndShutdownJoins(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var first, releaseOnce sync.Once + v := newTransactionVerifier(1, 8, func(*solana.Transaction) error { + first.Do(func() { close(started) }) + <-release + return nil + }) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + payload := prefetchTestPayload(t, verifierSignedTransactions(t, 1)) + for i := 0; i < entryPrefetchSlots+1; i++ { + batches := prefetchTestShreds(t, uint64(200+i), payload, buildAlpenglowEndingTick(t)) + admitted := make(chan struct{}) + go func() { + defer close(admitted) + for _, sh := range batches[0] { + _, err := a.addShredFrom(sh, false) + if err != nil { + t.Errorf("admit shred: %v", err) + } + } + }() + waitSignal(t, admitted, "nonblocking ingress while prefetch saturated") + } + waitSignal(t, started, "blocked verifier") + a.mu.Lock() + require.Equal(t, entryPrefetchSlots, p.slots) + require.LessOrEqual(t, p.bytes, entryPrefetchBytes) + require.Nil(t, a.slots[200+entryPrefetchSlots].prefetch) + a.mu.Unlock() + done := make(chan struct{}) + go func() { p.closeAndWait(); close(done) }() + select { + case <-done: + t.Fatal("early pool released buffers before signature worker joined") + case <-time.After(20 * time.Millisecond): + } + releaseOnce.Do(func() { close(release) }) + waitSignal(t, done, "saturated pool shutdown") + a.mu.Lock() + require.Zero(t, p.slots) + require.Zero(t, p.bytes) + require.Nil(t, a.entryPrefetch) + a.mu.Unlock() +} + +func TestEntryPrefetchInvalidRetainedTransactionFailsClosed(t *testing.T) { + txs := verifierSignedTransactions(t, 3) + txs[1].Signatures[0][3] ^= 0x80 + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + batches := prefetchTestShreds(t, 300, prefetchTestPayload(t, txs), buildAlpenglowEndingTick(t)) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + cached := waitPrefetchedBatch(t, a, 300, 0) + index, err := cached.verification.wait() + require.Equal(t, 1, index) + require.ErrorContains(t, err, "invalid signature") + var finalErr error + for _, sh := range batches[1] { + blk, err := a.AddShred(sh) + require.Nil(t, blk) + if err != nil { + finalErr = err + } + } + require.ErrorContains(t, finalErr, "transaction 1") + require.False(t, a.SlotCompleted(300)) +} + +func TestEntryPrefetchUpdateParentDiscardsInvalidOptimisticPrefix(t *testing.T) { + txs := verifierSignedTransactions(t, 2) + txs[0].Signatures[0][3] ^= 0x80 + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + const slot = 304 + batches := prefetchTestShreds(t, slot, + prefetchTestPayload(t, txs[:1]), + testAlpenglowParentMarkerBytes(blockMarkerVariantUpdateParent, slot-2, solana.Hash{12}), + prefetchTestPayload(t, txs[1:])) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + cached := waitPrefetchedBatch(t, a, slot, 0) + _, err := cached.verification.wait() + require.ErrorContains(t, err, "invalid signature") + require.Nil(t, feedPrefetchShreds(t, a, batches[1])) + blk := feedPrefetchShreds(t, a, batches[2]) + require.NotNil(t, blk) + require.Len(t, blk.Transactions, 1) + require.Equal(t, txs[1].Signatures[0], blk.Transactions[0].Signatures[0]) + require.Equal(t, uint64(slot-2), blk.SourceParentSlot) + require.True(t, blk.TransactionSignaturesVerified()) +} + +func TestEntryPrefetchCanceledCompletionCanRetrySameGeneration(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var first, releaseOnce sync.Once + v := newTransactionVerifier(1, 8, func(tx *solana.Transaction) error { + first.Do(func() { close(started); <-release }) + return txverify.VerifyTransaction(tx) + }) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + const slot = 308 + batches := prefetchTestShreds(t, slot, + prefetchTestPayload(t, verifierSignedTransactions(t, 3)), buildAlpenglowEndingTick(t)) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + waitSignal(t, started, "early verification") + cached := waitPrefetchedBatch(t, a, slot, 0) + var work *slotCompletionWork + for _, sh := range batches[1] { + var err error + work, err = a.addShredFrom(sh, false) + require.NoError(t, err) + } + require.NotNil(t, work) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + canceled := make(chan struct{}) + originalCancel := cached.verification.cancel + cached.verification.cancel = func() { close(canceled); originalCancel() } + done := make(chan processedSlotCompletion, 1) + go func() { done <- a.processCompletion(ctx, work) }() + // The completion has no expensive decode left and blocks joining this + // one already-prepared future; cancel while it owns those transactions. + time.Sleep(20 * time.Millisecond) + cancel() + waitSignal(t, canceled, "completion canceled its signature request") + releaseOnce.Do(func() { close(release) }) + var processed processedSlotCompletion + select { + case processed = <-done: + case <-time.After(3 * time.Second): + t.Fatal("canceled completion failed to join") + } + require.True(t, processed.canceled) + _, err := a.finalizeCompletion(work, processed) + require.NoError(t, err) + _, err = cached.verification.wait() + require.True(t, errors.Is(err, context.Canceled)) + a.mu.Lock() + retry := a.claimCompletionLocked(a.slots[slot], false) + a.mu.Unlock() + require.NotNil(t, retry) + processed = a.processCompletion(context.Background(), retry) + require.NoError(t, processed.err) + blk, err := a.finalizeCompletion(retry, processed) + require.NoError(t, err) + require.NotNil(t, blk) + require.True(t, blk.TransactionSignaturesVerified()) +} + +// A complete later batch must verify while an earlier batch still has a gap. +// Its preceding DATA_COMPLETE shred remains necessary to establish its start. +func TestEntryPrefetchBypassesEarlierGap(t *testing.T) { + for _, lateBoundary := range []bool{false, true} { + t.Run(fmt.Sprint("lateBoundary=", lateBoundary), func(t *testing.T) { + var calls atomic.Int32 + v := newTransactionVerifier(2, 8, func(tx *solana.Transaction) error { calls.Add(1); return txverify.VerifyTransaction(tx) }) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + const slot = 909 + txs := verifierSignedTransactions(t, 60) + batches := prefetchTestShreds(t, slot, prefetchTestPayload(t, txs[:30]), prefetchTestPayload(t, txs[30:]), buildAlpenglowEndingTick(t)) + require.Greater(t, len(batches[0]), 2) + end := len(batches[0]) - 1 + for i, sh := range batches[0] { + if i == 1 || (lateBoundary && i == end) { + continue + } + require.Nil(t, feedPrefetchShreds(t, a, []*Shred{sh})) + } + require.Nil(t, feedPrefetchShreds(t, a, batches[1])) + if lateBoundary { + a.mu.Lock() + empty := len(a.slots[slot].completeBatches) == 0 + a.mu.Unlock() + require.True(t, empty, "unknown preceding boundary must prevent speculation") + require.Nil(t, feedPrefetchShreds(t, a, batches[0][end:])) + } + later := waitPrefetchedBatch(t, a, slot, batches[1][0].Index) + _, err := later.verification.wait() + require.NoError(t, err) + require.Equal(t, int32(30), calls.Load()) + require.Nil(t, feedPrefetchShreds(t, a, batches[0][1:2])) + earlier := waitPrefetchedBatch(t, a, slot, 0) + _, err = earlier.verification.wait() + require.NoError(t, err) + blk := feedPrefetchShreds(t, a, batches[2]) + require.NotNil(t, blk) + require.True(t, blk.TransactionSignaturesVerified()) + require.Len(t, blk.Transactions, 60) + require.Equal(t, int32(60), calls.Load(), "every signature verified exactly once") + for i, tx := range txs { + require.Equal(t, tx.Signatures[0], blk.Transactions[i].Signatures[0]) + } + }) + } +} + +func TestEntryPrefetchDiscoversRecoveredBoundary(t *testing.T) { + v := newTransactionVerifier(2, 8, nil) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + const slot = 910 + txs := verifierSignedTransactions(t, 60) + gen := ShredGenerator{Slot: slot, ParentSlot: slot - 1, Version: 1} + raw := prefetchTestPayload(t, txs[:30]) + packets, root, nextData, nextCode, err := gen.MakeShredsFromData(testShredLeader(t), raw, false, solana.Hash{}, 0, 0) + require.NoError(t, err) + var code []*Shred + for _, packet := range packets { + sh, err := ParseShred(packet) + require.NoError(t, err) + if sh.Type == ShredTypeCode { + code = append(code, sh) + continue + } + if !sh.DataComplete() { + require.Nil(t, feedPrefetchShreds(t, a, []*Shred{sh})) + } + } + packets, _, _, _, err = gen.MakeShredsFromData(testShredLeader(t), prefetchTestPayload(t, txs[30:]), false, root, nextData, nextCode) + require.NoError(t, err) + for _, packet := range packets { + sh, err := ParseShred(packet) + require.NoError(t, err) + if sh.Type == ShredTypeData { + require.Nil(t, feedPrefetchShreds(t, a, []*Shred{sh})) + } + } + a.mu.Lock() + empty := len(a.slots[slot].completeBatches) == 0 + a.mu.Unlock() + require.True(t, empty) + require.NotEmpty(t, code) + for _, sh := range code { + require.Nil(t, feedPrefetchShreds(t, a, []*Shred{sh})) + } + later := waitPrefetchedBatch(t, a, slot, nextData) + _, err = later.verification.wait() + require.NoError(t, err) + first := waitPrefetchedBatch(t, a, slot, 0) + _, err = first.verification.wait() + require.NoError(t, err) +} + +func TestEntryPrefetchIndexDisabledAndLateInstall(t *testing.T) { + a := NewSlotAssembler() + batches := prefetchTestShreds(t, 911, prefetchTestPayload(t, verifierSignedTransactions(t, 3)), buildAlpenglowEndingTick(t)) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + a.mu.Lock() + absent := a.slots[911].batchIndex == nil + a.mu.Unlock() + require.True(t, absent) + v := newTransactionVerifier(2, 8, nil) + defer v.closeAndWait() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + // Seeding also works on a coding-only admission with no new data recovery. + a.mu.Lock() + a.prefetchEntriesLocked(a.slots[911]) + a.mu.Unlock() + cached := waitPrefetchedBatch(t, a, 911, 0) + _, err := cached.verification.wait() + require.NoError(t, err) +} + +func TestEntryPrefetchResetRetainsQueuedReservations(t *testing.T) { + a := NewSlotAssembler() + ctx, cancel := context.WithCancel(context.Background()) + // Hold workers until after reset so all tokens remain in the channel. + p := &entryPrefetchPool{a: a, ctx: ctx, cancel: cancel, jobs: make(chan *slotState, entryPrefetchSlots)} + a.entryPrefetch = p + startWorker := sync.OnceFunc(func() { + p.workers.Add(1) + go p.run() + }) + defer func() { + startWorker() + p.closeAndWait() + }() + for i := 0; i < entryPrefetchSlots; i++ { + s := &slotState{slot: uint64(i), completeBatches: []shredBatchRange{{0, 0}}} + a.mu.Lock() + a.slots[s.slot] = s + a.prefetchEntriesLocked(s) + a.releasePrefetchLocked(s) + a.mu.Unlock() + } + require.Equal(t, entryPrefetchSlots, len(p.jobs)) + // Cleanup must not admit another generation while stale queue tokens live. + require.Never(t, func() bool { + a.mu.Lock() + defer a.mu.Unlock() + return p.slots != entryPrefetchSlots + }, 50*time.Millisecond, time.Millisecond) + fresh := &slotState{slot: 100, completeBatches: []shredBatchRange{{0, 0}}} + a.mu.Lock() + a.prefetchEntriesLocked(fresh) + reserved := fresh.prefetch != nil + a.mu.Unlock() + // Start the worker before assertions so test failures cannot strand cleanup. + startWorker() + require.False(t, reserved) + p.cleanup.Wait() + a.mu.Lock() + slots := p.slots + a.mu.Unlock() + require.Zero(t, slots) +} diff --git a/pkg/turbine/fec_root_cache.go b/pkg/turbine/fec_root_cache.go new file mode 100644 index 000000000..cb4395b01 --- /dev/null +++ b/pkg/turbine/fec_root_cache.go @@ -0,0 +1,56 @@ +package turbine + +import ( + "bytes" + + "github.com/gagliardetto/solana-go" +) + +// One snapshot per FEC generation binds an authenticated root to its source. +// It is written only during assembly, before the state is frozen. Completion +// never trusts pointer identity alone or changes the deterministic root choice. +type authenticatedFECRoot struct { + source *Shred + input shredRootInput + payload []byte + root solana.Hash +} + +// These are every parsed field read by MerkleRoot. The exact payload comparison +// covers headers, data, proof, and trailers, even for noncanonical test input. +type shredRootInput struct { + variant byte + kind ShredType + index, fecSetIndex uint32 + dataCount, position uint16 +} + +func rootInput(s *Shred) shredRootInput { + return shredRootInput{s.Variant, s.Type, s.Index, s.FECSetIndex, s.NumDataShreds, s.Position} +} + +func hasMerkleRootProof(s *Shred) bool { + return s != nil && isMerkleVariant(s.Variant) && (s.Type == ShredTypeData || s.Type == ShredTypeCode) +} + +func (c *authenticatedFECRoot) matches(s *Shred) bool { + return s == c.source && rootInput(s) == c.input && bytes.Equal(s.Payload, c.payload) +} + +func (f *fecState) rememberAuthenticatedRoot(s *Shred, root solana.Hash) { + if s.Recovered || !isMerkleVariant(s.Variant) { + return + } + if cached := f.rootCache; cached != nil { + // Data proofs precede coding proofs, and the lowest index wins. An + // unauthenticated earlier arrival can still force fallback at completion. + old := cached.input + if old.kind == ShredTypeData && (s.Type != ShredTypeData || s.Index >= old.index) { + return + } + if old.kind == ShredTypeCode && s.Type == ShredTypeCode && s.Position >= old.position { + return + } + } + f.rootCache = &authenticatedFECRoot{source: s, input: rootInput(s), payload: bytes.Clone(s.Payload), root: root} +} diff --git a/pkg/turbine/generate.go b/pkg/turbine/generate.go index 14626e214..f8a4e6343 100644 --- a/pkg/turbine/generate.go +++ b/pkg/turbine/generate.go @@ -4,6 +4,7 @@ import ( "crypto/ed25519" "encoding/binary" "fmt" + "sync" "github.com/gagliardetto/solana-go" "github.com/klauspost/reedsolomon" @@ -14,6 +15,21 @@ const ( proofEntriesFor32x32 = 6 ) +var erasureEncoderPool sync.Pool + +func acquireErasureEncoder() (reedsolomon.Encoder, error) { + if encoder := erasureEncoderPool.Get(); encoder != nil { + return encoder.(reedsolomon.Encoder), nil + } + return reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) +} + +func releaseErasureEncoder(encoder reedsolomon.Encoder) { + if encoder != nil { + erasureEncoderPool.Put(encoder) + } +} + // ShredGenerator builds merkle FEC shreds from a serialized byte buffer. type ShredGenerator struct { Slot uint64 @@ -22,6 +38,14 @@ type ShredGenerator struct { ReferenceTick uint8 } +// shredPackets retains the roots already computed during generation, in FEC +// order, so broadcast can commit to them without parsing the packets again. +type shredPackets struct { + packets [][]byte + fecSetRoots []solana.Hash + chainedMerkleRoot solana.Hash +} + func dataCapacity(proofSize uint8, resigned bool) int { capacity := dataPayloadSize - dataHeaderSize - merkleRootSize - int(proofSize)*merkleProofEntrySize if resigned { @@ -66,9 +90,31 @@ func (g *ShredGenerator) MakeShredsFromData( nextShredIndex uint32, nextCodeIndex uint32, ) ([][]byte, solana.Hash, uint32, uint32, error) { + batch, nextData, nextCode, err := g.makeShredsFromData( + leader, data, isLastInSlot, chainedMerkleRoot, nextShredIndex, nextCodeIndex, + ) + return batch.packets, batch.chainedMerkleRoot, nextData, nextCode, err +} + +func (g *ShredGenerator) makeShredsFromData( + leader solana.PrivateKey, + data []byte, + isLastInSlot bool, + chainedMerkleRoot solana.Hash, + nextShredIndex uint32, + nextCodeIndex uint32, +) (shredPackets, uint32, uint32, error) { if g.Slot < g.ParentSlot || g.Slot-g.ParentSlot > uint64(^uint16(0)) { - return nil, solana.Hash{}, nextShredIndex, nextCodeIndex, fmt.Errorf("invalid parent slot %d for slot %d", g.ParentSlot, g.Slot) + return shredPackets{}, nextShredIndex, nextCodeIndex, fmt.Errorf("invalid parent slot %d for slot %d", g.ParentSlot, g.Slot) + } + // The 32+32 coding matrix is invariant across every FEC set in this + // operation. Building it requires a Vandermonde inversion, so retain the + // encoder for the whole payload rather than reconstructing it per set. + encoder, err := acquireErasureEncoder() + if err != nil { + return shredPackets{}, nextShredIndex, nextCodeIndex, err } + defer releaseErasureEncoder(encoder) proofSize := uint8(proofEntriesFor32x32) unsignedCap := dataCapacity(proofSize, false) signedCap := dataCapacity(proofSize, true) @@ -92,49 +138,62 @@ func (g *ShredGenerator) MakeShredsFromData( } var packets [][]byte + var fecSetRoots []solana.Hash dataIndex := nextShredIndex codeIndex := nextCodeIndex chainedRoot := chainedMerkleRoot + // DATA_COMPLETE ends the serialized component, which may span FEC sets. for len(unsignedData) >= unsignedBatch { batch := unsignedData[:unsignedBatch] unsignedData = unsignedData[unsignedBatch:] - batchPackets, root, err := g.makeFECBatch(leader, batch, unsignedCap, proofSize, false, parentOffset, flags, false, chainedRoot, dataIndex, codeIndex) + // DATA_COMPLETE marks the end of the serialized component, not the end + // of every FEC set. A full unsigned batch is complete only when no + // unsigned remainder or signed-last batch follows it. + dataComplete := len(unsignedData) == 0 && len(signedData) == 0 + batchPackets, root, err := g.makeFECBatch(encoder, leader, batch, unsignedCap, proofSize, false, parentOffset, flags, dataComplete, false, chainedRoot, dataIndex, codeIndex) if err != nil { - return nil, solana.Hash{}, dataIndex, codeIndex, err + return shredPackets{}, dataIndex, codeIndex, err } packets = append(packets, batchPackets...) + fecSetRoots = append(fecSetRoots, root) chainedRoot = root dataIndex += dataShredsPerFECBlock codeIndex += codingShredsPerFECBlock } if len(unsignedData) > 0 || (len(packets) == 0 && !isLastInSlot) { - batchPackets, root, err := g.makeFECBatch(leader, unsignedData, unsignedCap, proofSize, false, parentOffset, flags, false, chainedRoot, dataIndex, codeIndex) + dataComplete := len(signedData) == 0 + batchPackets, root, err := g.makeFECBatch(encoder, leader, unsignedData, unsignedCap, proofSize, false, parentOffset, flags, dataComplete, false, chainedRoot, dataIndex, codeIndex) if err != nil { - return nil, solana.Hash{}, dataIndex, codeIndex, err + return shredPackets{}, dataIndex, codeIndex, err } packets = append(packets, batchPackets...) + fecSetRoots = append(fecSetRoots, root) chainedRoot = root dataIndex += dataShredsPerFECBlock codeIndex += codingShredsPerFECBlock } if len(signedData) > 0 || (len(packets) == 0 && isLastInSlot) { - batchPackets, root, err := g.makeFECBatch(leader, signedData, signedCap, proofSize, true, parentOffset, flags, isLastInSlot, chainedRoot, dataIndex, codeIndex) + batchPackets, root, err := g.makeFECBatch(encoder, leader, signedData, signedCap, proofSize, true, parentOffset, flags, true, isLastInSlot, chainedRoot, dataIndex, codeIndex) if err != nil { - return nil, solana.Hash{}, dataIndex, codeIndex, err + return shredPackets{}, dataIndex, codeIndex, err } packets = append(packets, batchPackets...) + fecSetRoots = append(fecSetRoots, root) chainedRoot = root dataIndex += dataShredsPerFECBlock codeIndex += codingShredsPerFECBlock } - return packets, chainedRoot, dataIndex, codeIndex, nil + return shredPackets{ + packets: packets, fecSetRoots: fecSetRoots, chainedMerkleRoot: chainedRoot, + }, dataIndex, codeIndex, nil } func (g *ShredGenerator) makeFECBatch( + encoder reedsolomon.Encoder, leader solana.PrivateKey, data []byte, dataCap int, @@ -142,6 +201,7 @@ func (g *ShredGenerator) makeFECBatch( resigned bool, parentOffset uint16, flags byte, + dataComplete bool, isLastInSlot bool, chainedMerkleRoot solana.Hash, dataIndex uint32, @@ -196,11 +256,11 @@ func (g *ShredGenerator) makeFECBatch( dataPackets[i][dataFlagsOffset] |= shredFlagLastShredInSlot break } - } else if len(dataPackets) > 0 { + } else if dataComplete && len(dataPackets) > 0 { dataPackets[len(dataPackets)-1][dataFlagsOffset] |= shredFlagDataComplete } - root, err := finishErasureBatch(leader, allPackets, chainedMerkleRoot, proofSize, resigned) + root, err := finishErasureBatch(encoder, leader, allPackets, chainedMerkleRoot, proofSize, resigned) if err != nil { return nil, solana.Hash{}, err } @@ -208,101 +268,70 @@ func (g *ShredGenerator) makeFECBatch( } func finishErasureBatch( + encoder reedsolomon.Encoder, leader solana.PrivateKey, packets [][]byte, chainedMerkleRoot solana.Hash, proofSize uint8, resigned bool, ) (solana.Hash, error) { - encoder, err := reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) + if len(packets) != dataShredsPerFECBlock+codingShredsPerFECBlock { + return solana.Hash{}, fmt.Errorf("invalid FEC packet count %d", len(packets)) + } + dataCap, err := merkleCapacity(dataPayloadSize, dataHeaderSize, proofSize, true, resigned) if err != nil { return solana.Hash{}, err } + codeCap, err := merkleCapacity(codingPayloadSize, codingHeaderSize, proofSize, true, resigned) + if err != nil { + return solana.Hash{}, err + } + dataVariant := chainedDataVariant(proofSize, resigned) + codeVariant := chainedCodeVariant(proofSize, resigned) + // These packets were constructed immediately above, so retain direct views + // of their erasure regions. ParseShred is intentionally a defensive, + // owning parser for untrusted network packets; using it here would allocate + // and copy every packet several times only to copy the same bytes back. shards := make([][]byte, len(packets)) for i, packet := range packets { - shred, err := ParseShred(packet) - if err != nil { - return solana.Hash{}, fmt.Errorf("parse batch shred %d: %w", i, err) + if i < dataShredsPerFECBlock { + if len(packet) < dataPayloadSize || packet[shredVariantOffset] != dataVariant { + return solana.Hash{}, fmt.Errorf("invalid generated data shred %d", i) + } + shards[i] = packet[shredSignatureSize : dataHeaderSize+dataCap] + continue } - shard, err := shred.erasureShard() - if err != nil { - return solana.Hash{}, fmt.Errorf("erasure shard %d: %w", i, err) + if len(packet) < codingPayloadSize || packet[shredVariantOffset] != codeVariant { + return solana.Hash{}, fmt.Errorf("invalid generated coding shred %d", i-dataShredsPerFECBlock) } - shards[i] = shard + shards[i] = packet[codingHeaderSize : codingHeaderSize+codeCap] } if err := encoder.Encode(shards); err != nil { return solana.Hash{}, fmt.Errorf("reed-solomon encode: %w", err) } for i, packet := range packets { - shred, err := ParseShred(packet) - if err != nil { - return solana.Hash{}, err - } - proofSizeInfo, chained, resignedFlag, ok := merkleVariantInfo(shred.Variant) - if !ok { - return solana.Hash{}, ErrUnsupportedShred - } - _ = proofSizeInfo - _ = chained - _ = resignedFlag - - capacity, err := merkleCapacity(len(packet), dataHeaderSize, proofSize, true, resigned) - if shred.Type == ShredTypeCode { - capacity, err = merkleCapacity(len(packet), codingHeaderSize, proofSize, true, resigned) - } - if err != nil { - return solana.Hash{}, err - } - rootOffset := dataHeaderSize + capacity - if shred.Type == ShredTypeCode { - rootOffset = codingHeaderSize + capacity + rootOffset := dataHeaderSize + dataCap + if i >= dataShredsPerFECBlock { + rootOffset = codingHeaderSize + codeCap } copy(packet[rootOffset:rootOffset+merkleRootSize], chainedMerkleRoot[:]) - - if shred.Type == ShredTypeCode { - start := codingHeaderSize - end := start + capacity - copy(packet[start:end], shards[i]) - } else { - start := shredSignatureSize - end := dataHeaderSize + capacity - copy(packet[start:end], shards[i]) - } } - nodes, err := buildMerkleTree(packets) - if err != nil { - return solana.Hash{}, err - } + nodes := buildGeneratedMerkleTree(packets, dataCap, codeCap) root := nodes[len(nodes)-1] sig := ed25519.Sign(ed25519.PrivateKey(leader), root[:]) - for _, packet := range packets { + for i, packet := range packets { copy(packet[shredSignatureOffset:shredSignatureSize], sig) - shred, err := ParseShred(packet) - if err != nil { - return solana.Hash{}, err - } - leafIndex, err := shred.merkleLeafIndex() - if err != nil { - return solana.Hash{}, err - } - proof := makeMerkleProof(nodes, leafIndex, len(packets)) - capacity, err := merkleCapacity(len(packet), dataHeaderSize, proofSize, true, resigned) - if shred.Type == ShredTypeCode { - capacity, err = merkleCapacity(len(packet), codingHeaderSize, proofSize, true, resigned) - } - if err != nil { - return solana.Hash{}, err - } - proofOffset := dataHeaderSize + capacity + merkleRootSize - if shred.Type == ShredTypeCode { - proofOffset = codingHeaderSize + capacity + merkleRootSize + proofOffset := dataHeaderSize + dataCap + merkleRootSize + if i >= dataShredsPerFECBlock { + proofOffset = codingHeaderSize + codeCap + merkleRootSize } - for j, entry := range proof { - copy(packet[proofOffset+j*merkleProofEntrySize:], entry[:]) + proofEntries := writeMerkleProof(packet[proofOffset:], nodes, i, len(packets)) + if proofEntries != int(proofSize) { + return solana.Hash{}, fmt.Errorf("generated merkle proof has %d entries, want %d", proofEntries, proofSize) } if resigned { retransmitOffset := proofOffset + int(proofSize)*merkleProofEntrySize @@ -312,6 +341,54 @@ func finishErasureBatch( return root, nil } +// buildGeneratedMerkleTree hashes the fixed packet order emitted by +// makeFECBatch: 32 data shreds followed by 32 coding shreds. Callers must have +// already validated the packet sizes and variants in finishErasureBatch. +func buildGeneratedMerkleTree(packets [][]byte, dataCap, codeCap int) []solana.Hash { + leaves := make([]solana.Hash, len(packets)) + for i, packet := range packets { + end := dataHeaderSize + dataCap + merkleRootSize + if i >= dataShredsPerFECBlock { + end = codingHeaderSize + codeCap + merkleRootSize + } + leaves[i] = merkleHashLeaf(packet[shredSignatureSize:end]) + } + + nodes := make([]solana.Hash, 0, merkleTreeSize(len(leaves))) + nodes = append(nodes, leaves...) + for size := len(leaves); size > 1; size = (size + 1) >> 1 { + offset := len(nodes) - size + for index := offset; index < offset+size; index += 2 { + other := index + 1 + if other >= offset+size { + other = offset + size - 1 + } + nodes = append(nodes, merkleHashNode(nodes[index][:merkleProofEntrySize], nodes[other][:merkleProofEntrySize])) + } + } + return nodes +} + +// writeMerkleProof writes the truncated sibling hashes directly into a packet. +// The generated FEC tree has fixed depth, so materializing a temporary proof +// slice for every one of its 64 packets only adds allocator and copy traffic. +func writeMerkleProof(dst []byte, nodes []solana.Hash, index, size int) int { + entries := 0 + offset := 0 + for size > 1 { + sibling := index ^ 1 + if sibling >= size { + sibling = size - 1 + } + copy(dst[entries*merkleProofEntrySize:], nodes[offset+sibling][:merkleProofEntrySize]) + entries++ + offset += size + size = (size + 1) >> 1 + index >>= 1 + } + return entries +} + func buildMerkleTree(packets [][]byte) ([]solana.Hash, error) { leaves := make([]solana.Hash, len(packets)) for i, packet := range packets { diff --git a/pkg/turbine/generate_bench_test.go b/pkg/turbine/generate_bench_test.go new file mode 100644 index 000000000..2301eefaa --- /dev/null +++ b/pkg/turbine/generate_bench_test.go @@ -0,0 +1,242 @@ +package turbine + +import ( + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/gagliardetto/solana-go" + "github.com/klauspost/reedsolomon" +) + +const ( + benchmarkTransactionCount = 50_000 + benchmarkTransactionBytes = 1_232 +) + +var ( + benchmarkEncoderSink reedsolomon.Encoder + benchmarkPacketsSink [][]byte + benchmarkRootSink solana.Hash + benchmarkByteSink byte +) + +func benchmarkLeaderKey() solana.PrivateKey { + var seed [ed25519.SeedSize]byte + for i := range seed { + seed[i] = byte(i + 1) + } + return solana.PrivateKey(ed25519.NewKeyFromSeed(seed[:])) +} + +func benchmarkPayload(size int) []byte { + payload := make([]byte, size) + var state uint64 = 0x9e3779b97f4a7c15 + for i := range payload { + // A deterministic, non-zero corpus avoids accidentally benchmarking a + // special all-zero input while keeping fixture construction out of the + // timed region. + state ^= state << 7 + state ^= state >> 9 + state ^= state << 8 + payload[i] = byte(state) + } + return payload +} + +// BenchmarkReedSolomonEncode32x32 isolates the arithmetic kernel used by one +// unsigned 32+32 chained FEC set. Encoder construction, shred parsing, Merkle +// hashing, signing, and packet copies are intentionally outside this result. +func BenchmarkReedSolomonEncode32x32(b *testing.B) { + const shardBytes = 987 // unsigned chained 32+32 shreds with proof size 6 + encoder, err := reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) + if err != nil { + b.Fatal(err) + } + shards := make([][]byte, dataShredsPerFECBlock+codingShredsPerFECBlock) + for i := range shards { + shards[i] = make([]byte, shardBytes) + if i < dataShredsPerFECBlock { + copy(shards[i], benchmarkPayload(shardBytes)) + } + } + + b.SetBytes(dataShredsPerFECBlock * shardBytes) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := encoder.Encode(shards); err != nil { + b.Fatal(err) + } + } + benchmarkByteSink = shards[len(shards)-1][shardBytes-1] +} + +// BenchmarkReedSolomonNew32x32 measures work that finishErasureBatch currently +// repeats for every FEC set even though the 32+32 shape never changes. +func BenchmarkReedSolomonNew32x32(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + encoder, err := reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) + if err != nil { + b.Fatal(err) + } + benchmarkEncoderSink = encoder + } +} + +// BenchmarkMakeShredsFromData reports the complete current generator cost, +// including packet construction, Reed-Solomon coding, chained Merkle trees, +// one Ed25519 signature per FEC set, and proof materialization. +func BenchmarkMakeShredsFromData(b *testing.B) { + const blockBytes = benchmarkTransactionCount * benchmarkTransactionBytes + cases := []struct { + name string + size int + isLastInSlot bool + }{ + {name: "one-unsigned-fec", size: dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false)}, + {name: "block-50000x1232", size: blockBytes, isLastInSlot: true}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + leader := benchmarkLeaderKey() + payload := benchmarkPayload(tc.size) + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1} + + b.SetBytes(int64(tc.size)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + packets, root, _, _, err := gen.MakeShredsFromData(leader, payload, tc.isLastInSlot, solana.Hash{}, 0, 0) + if err != nil { + b.Fatal(err) + } + benchmarkPacketsSink = packets + benchmarkRootSink = root + } + b.StopTimer() + if len(benchmarkPacketsSink) == 0 || len(benchmarkPacketsSink)%64 != 0 { + b.Fatalf("unexpected packet count %d", len(benchmarkPacketsSink)) + } + b.ReportMetric(float64(len(benchmarkPacketsSink)), "packets/op") + b.ReportMetric(float64(len(benchmarkPacketsSink)/64), "FEC-sets/op") + if tc.size == blockBytes { + b.ReportMetric(benchmarkTransactionCount, "transactions/op") + } + }) + } +} + +// BenchmarkMakeShreds50000TargetBatches1232 models the producer's target-sized +// component stream without retaining a multi-gigabyte output. It measures only +// the 61.6 MB transaction payload; entry framing is deliberately outside this +// erasure-coding benchmark. +func BenchmarkMakeShreds50000TargetBatches1232(b *testing.B) { + const inputBytes = benchmarkTransactionCount * benchmarkTransactionBytes + leader := benchmarkLeaderKey() + payload := benchmarkPayload(inputBytes) + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1} + + b.SetBytes(inputBytes) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var ( + root solana.Hash + dataIndex uint32 + codeIndex uint32 + ) + var ( + offset int + components int + packetCount int + fecSetCount int + ) + for offset < len(payload) { + end := min(offset+costmodel.DefaultTargetBatchBytes, len(payload)) + packets, nextRoot, nextData, nextCode, err := gen.MakeShredsFromData( + leader, + payload[offset:end], + end == len(payload), + root, + dataIndex, + codeIndex, + ) + if err != nil { + b.Fatal(err) + } + benchmarkPacketsSink = packets + components++ + packetCount += len(packets) + fecSetCount += len(packets) / (dataShredsPerFECBlock + codingShredsPerFECBlock) + root, dataIndex, codeIndex = nextRoot, nextData, nextCode + offset = end + } + b.ReportMetric(float64(components), "components/op") + b.ReportMetric(float64(fecSetCount), "FEC-sets/op") + b.ReportMetric(float64(packetCount), "packets/op") + benchmarkRootSink = root + } + b.StopTimer() + b.ReportMetric(benchmarkTransactionCount, "transactions/op") +} + +func TestBenchmarkBlockPayloadAccounting(t *testing.T) { + const blockBytes = benchmarkTransactionCount * benchmarkTransactionBytes + unsignedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + unsignedBytes := blockBytes - signedBatch + unsignedFECs := (unsignedBytes + unsignedBatch - 1) / unsignedBatch + totalFECs := unsignedFECs + 1 + if totalFECs != 2000 { + t.Fatalf("50k x 1232 payload maps to %d FEC sets, want 2000 (%s)", totalFECs, fmt.Sprintf("%d bytes", blockBytes)) + } +} + +func TestProducerTargetMatchesTwoTypicalFECPayloads(t *testing.T) { + want := 2 * dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + if costmodel.DefaultTargetBatchBytes != want { + t.Fatalf("producer target = %d, want two typical FEC payloads = %d", costmodel.DefaultTargetBatchBytes, want) + } +} + +func TestMakeShredsFromDataStableBytes(t *testing.T) { + unsignedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + tests := []struct { + name string + size int + isLastInSlot bool + want string + }{ + {name: "unsigned-one-fec", size: unsignedBatch, want: "f9334f1835240df21d4b48a09f35b3ff90578122d30d0527608c10d38d0911f7"}, + {name: "signed-one-fec", size: signedBatch, isLastInSlot: true, want: "bfa398c445509c5e1345553bbe84fea04e86001caf441008b4014d07c6e36ccd"}, + {name: "two-unsigned-one-signed", size: 2*unsignedBatch + signedBatch, isLastInSlot: true, want: "dffae840e4c247680b5e1667747a63138872a0080c51a6fcf4cc5002eb7778ac"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1, ReferenceTick: 17} + packets, _, _, _, err := gen.MakeShredsFromData( + benchmarkLeaderKey(), benchmarkPayload(tt.size), tt.isLastInSlot, + solana.Hash{3}, 7, 11, + ) + if err != nil { + t.Fatal(err) + } + h := sha256.New() + for _, packet := range packets { + _, _ = h.Write(packet) + } + got := hex.EncodeToString(h.Sum(nil)) + if got != tt.want { + t.Fatalf("packet digest %s, want %s", got, tt.want) + } + }) + } +} diff --git a/pkg/turbine/generate_test.go b/pkg/turbine/generate_test.go index 083133954..64c5d1091 100644 --- a/pkg/turbine/generate_test.go +++ b/pkg/turbine/generate_test.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/ed25519" "encoding/binary" + "fmt" "testing" "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" @@ -174,6 +175,46 @@ func TestMakeShredsFromDataRoundTrip(t *testing.T) { } } +func TestGeneratedFECRootsMatchPacketProofs(t *testing.T) { + unsignedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + for _, last := range []bool{false, true} { + for _, size := range []int{0, 1, signedBatch, signedBatch + 1, unsignedBatch, unsignedBatch + 1, 2 * unsignedBatch, 2*unsignedBatch + signedBatch} { + t.Run(fmt.Sprintf("last=%t/bytes=%d", last, size), func(t *testing.T) { + gen := ShredGenerator{Slot: 100, ParentSlot: 99, Version: 7, ReferenceTick: 17} + parentRoot := solana.Hash{5} + leader := testShredLeader(t) + batch, nextData, nextCode, err := gen.makeShredsFromData( + leader, benchmarkPayload(size), last, parentRoot, 7, 11, + ) + require.NoError(t, err) + require.NotEmpty(t, batch.fecSetRoots) + require.Len(t, batch.packets, len(batch.fecSetRoots)*(dataShredsPerFECBlock+codingShredsPerFECBlock)) + require.Equal(t, uint32(7+len(batch.fecSetRoots)*dataShredsPerFECBlock), nextData) + require.Equal(t, uint32(11+len(batch.fecSetRoots)*codingShredsPerFECBlock), nextCode) + require.Equal(t, batch.fecSetRoots[len(batch.fecSetRoots)-1], batch.chainedMerkleRoot) + for i, packet := range batch.packets { + fec := i / (dataShredsPerFECBlock + codingShredsPerFECBlock) + shred, err := ParseShred(packet) + require.NoError(t, err) + root, err := shred.MerkleRoot() + require.NoError(t, err) + require.Equal(t, batch.fecSetRoots[fec], root) + require.Equal(t, uint32(7+fec*dataShredsPerFECBlock), shred.FECSetIndex) + previousRoot := parentRoot + if fec > 0 { + previousRoot = batch.fecSetRoots[fec-1] + } + embeddedRoot, err := shred.EmbeddedChainedMerkleRoot() + require.NoError(t, err) + require.Equal(t, previousRoot, embeddedRoot) + require.NoError(t, shred.VerifySignature(leader.PublicKey())) + } + }) + } + } +} + func TestMakeShredsFromAlpenglowBlock(t *testing.T) { leader := testShredLeader(t) gen := ShredGenerator{ @@ -184,10 +225,10 @@ func TestMakeShredsFromAlpenglowBlock(t *testing.T) { } var ( - chainedRoot = solana.Hash{5} - nextData uint32 = 0 - nextCode uint32 = 0 - allDataShreds []*Shred + chainedRoot = solana.Hash{5} + nextData uint32 = 0 + nextCode uint32 = 0 + allDataShreds []*Shred ) for _, component := range buildAlpenglowSlot(t) { packets, root, newData, newCode, err := gen.MakeShredsFromData( diff --git a/pkg/turbine/generated_component_boundary_test.go b/pkg/turbine/generated_component_boundary_test.go new file mode 100644 index 000000000..871843089 --- /dev/null +++ b/pkg/turbine/generated_component_boundary_test.go @@ -0,0 +1,47 @@ +package turbine + +import ( + "bytes" + "fmt" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Exercise exact FEC boundaries and the smaller resigned final-set capacity. +// The receiver must see one DATA_COMPLETE per serialized component, regardless +// of how many FEC sets carry it, with proofs authenticating the final flags. +func TestGeneratedComponentHasOneCompletionBoundary(t *testing.T) { + unsigned := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signed := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + for _, last := range []bool{false, true} { + for _, size := range []int{0, 1, signed, signed + 1, unsigned, unsigned + 1, 2 * unsigned, 2*unsigned + signed} { + t.Run(fmt.Sprintf("last=%t/bytes=%d", last, size), func(t *testing.T) { + leader := testShredLeader(t) + gen := ShredGenerator{Slot: 100, ParentSlot: 99, Version: 7} + payload := bytes.Repeat([]byte{0x5a}, size) + packets, _, nextData, _, err := gen.MakeShredsFromData(leader, payload, last, solana.Hash{}, 0, 0) + require.NoError(t, err) + var decoded []byte + var completed int + for _, packet := range packets { + shred, err := ParseShred(packet) + require.NoError(t, err) + require.NoError(t, shred.VerifySignature(leader.PublicKey())) + if shred.Type != ShredTypeData { + continue + } + decoded = append(decoded, shred.Data...) + if shred.DataComplete() { + completed++ + require.Equal(t, nextData-1, shred.Index) + } + require.Equal(t, last && shred.Index == nextData-1, shred.LastInSlot()) + } + require.Equal(t, 1, completed) + require.True(t, bytes.Equal(payload, decoded)) + }) + } + } +} diff --git a/pkg/turbine/internal/rsrecover/doc.go b/pkg/turbine/internal/rsrecover/doc.go new file mode 100644 index 000000000..38bc6e7f5 --- /dev/null +++ b/pkg/turbine/internal/rsrecover/doc.go @@ -0,0 +1,5 @@ +// Package rsrecover contains fixed-shape Reed-Solomon recovery plans for +// Solana's 32 data + 32 coding shred FEC sets. Production dispatch uses only +// exactly-one-missing-data recovery. Reduced-subset and all-coding plans are +// reference/benchmark alternatives and are not selected by SlotAssembler. +package rsrecover diff --git a/pkg/turbine/internal/rsrecover/recover.go b/pkg/turbine/internal/rsrecover/recover.go new file mode 100644 index 000000000..6a16e91e9 --- /dev/null +++ b/pkg/turbine/internal/rsrecover/recover.go @@ -0,0 +1,599 @@ +package rsrecover + +import ( + "errors" + "fmt" + "math/bits" + "sync" + + "github.com/klauspost/reedsolomon" +) + +const ( + DataShards = 32 + CodingShards = 32 + TotalShards = DataShards + CodingShards + + cauchyGamma = byte(0xa5) +) + +var ( + ErrInvalidPattern = errors.New("invalid erasure pattern") + ErrPatternChanged = errors.New("availability differs from prepared plan") + ErrInvalidBuffers = errors.New("invalid recovery buffers") + + gfLog [256]byte + gfExp [512]byte + + allCodingEncoderOnce sync.Once + allCodingEncoder reedsolomon.Encoder + allCodingEncoderErr error + + // oneDataCoefficientRows[missing][coding] contains one coefficient for + // each data input followed by the selected coding input. Every one of the + // fixed 32x32 rows is exhaustively differential-tested against the general + // Reed-Solomon decoder. + oneDataCoefficientRows [DataShards][CodingShards][DataShards + 1]byte +) + +func init() { + value := uint16(1) + for exponent := 0; exponent < 255; exponent++ { + gfExp[exponent] = byte(value) + gfLog[byte(value)] = byte(exponent) + value <<= 1 + if value&0x100 != 0 { + value ^= 0x11d + } + } + for exponent := 255; exponent < len(gfExp); exponent++ { + gfExp[exponent] = gfExp[exponent-255] + } + for missing := 0; missing < DataShards; missing++ { + for coding := 0; coding < CodingShards; coding++ { + // If c = sum(a_i*d_i), then the missing d_m is + // inv(a_m) * (c + sum(i != m, a_i*d_i)) over GF(2^8). + coefficientInv := reedsolomon.Inv(codingCoefficient(coding, missing)) + for data := 0; data < DataShards; data++ { + if data != missing { + oneDataCoefficientRows[missing][coding][data] = gfMul( + coefficientInv, + codingCoefficient(coding, data), + ) + } + } + oneDataCoefficientRows[missing][coding][DataShards] = coefficientInv + } + } +} + +// OneDataPlan recovers exactly one absent data shard from the other 31 data +// shards and one available coding shard. It avoids a general 32x32 decode +// matrix inversion. The plan is immutable and safe for concurrent execution +// when callers provide independent destinations. +type OneDataPlan struct { + presence uint64 + missing uint8 + sources [DataShards]uint8 + coefficients [DataShards]byte +} + +// DataSubsetPlan recovers every absent data shard using the present data and +// the lowest-indexed coding rows required to reach the 32-shard threshold. +// Setup solves only the m x m system induced by the missing data columns. +type DataSubsetPlan struct { + presence uint64 + missing []uint8 + sources [DataShards]uint8 + weights [][]byte +} + +// AllCodingPlan recovers all 32 data rows from all 32 coding rows. For the +// fixed Solana matrix C*C=I, so the package's optimized encoder can apply C a +// second time instead of constructing a decode matrix. +type AllCodingPlan struct { + presence uint64 + encoder reedsolomon.Encoder +} + +// Presence reports which of the 64 input shards are non-empty. A zero-length +// shard is absent, matching reedsolomon.ReconstructSome. +func Presence(shards [][]byte) (uint64, error) { + if len(shards) != TotalShards { + return 0, fmt.Errorf("%w: got %d shards, want %d", ErrInvalidBuffers, len(shards), TotalShards) + } + var mask uint64 + for index, shard := range shards { + if len(shard) != 0 { + mask |= uint64(1) << index + } + } + return mask, nil +} + +// PrepareRecoverOneData constructs the direct coefficient row for a pattern +// with exactly one missing data shard. Additional coding shards may be present; +// the lowest-indexed one is selected deterministically. +func PrepareRecoverOneData(presence uint64, missingDataIndex int) (OneDataPlan, error) { + if missingDataIndex < 0 || missingDataIndex >= DataShards { + return OneDataPlan{}, fmt.Errorf("%w: missing data index %d", ErrInvalidPattern, missingDataIndex) + } + for index := 0; index < DataShards; index++ { + present := presence&(uint64(1)<= DataShards { + return fmt.Errorf("%w: missing data index %d", ErrInvalidPattern, missingDataIndex) + } + const dataMask = uint64(1)<> DataShards) + if codingMask == 0 { + return fmt.Errorf("%w: no coding shard is available", ErrInvalidPattern) + } + shardSize, err := validateExecution(presence, shards, [][]byte{dst}) + if err != nil { + return err + } + if len(dst) != shardSize { + return fmt.Errorf("%w: destination has %d bytes, want %d", ErrInvalidBuffers, len(dst), shardSize) + } + + codingPosition := bits.TrailingZeros32(codingMask) + row := &oneDataCoefficientRows[missingDataIndex][codingPosition] + var lowLevel reedsolomon.LowLevel + first := true + for dataIndex := 0; dataIndex < DataShards; dataIndex++ { + coefficient := row[dataIndex] + if coefficient == 0 { + continue + } + if first { + lowLevel.GalMulSlice(coefficient, shards[dataIndex], dst) + first = false + } else { + lowLevel.GalMulSliceXor(coefficient, shards[dataIndex], dst) + } + } + lowLevel.GalMulSliceXor(row[DataShards], shards[DataShards+codingPosition], dst) + return nil +} + +// PrepareRecoverDataSubset constructs direct output rows for all absent data +// shards. It first inverts only the reduced m x m coding/data matrix, then +// expands those rows over exactly 32 selected input shards so byte execution +// can be compared fairly with a general decoder. +func PrepareRecoverDataSubset(presence uint64) (DataSubsetPlan, error) { + plan := DataSubsetPlan{presence: presence} + knownData := make([]uint8, 0, DataShards) + for index := 0; index < DataShards; index++ { + if presence&(uint64(1)< code.Position { + code = s + } + } + if code == nil { + return fmt.Errorf("recover FEC: missing coding template") + } + proofSize, chained, _, ok := merkleVariantInfo(code.Variant) + if !ok || int(proofSize) != bits.Len(uint(len(shards)-1)) { + return fmt.Errorf("recover FEC: invalid Merkle proof depth") + } + if code.Index < uint32(code.Position) { + return fmt.Errorf("recover FEC: invalid coding index") + } + codeBase := code.Index - uint32(code.Position) + if uint64(codeBase)+uint64(f.layout.codingShreds)-1 > math.MaxUint32 { + return fmt.Errorf("recover FEC: coding index overflow") + } + expected, err := code.MerkleRoot() + if err != nil { + return err + } + encoder, err := a.fecEncoder(f.layout) + if err != nil { + return err + } + // Present shards are read-only. Only absent coding shards remain after the + // caller's data recovery, so this also checks commitments to missing parity. + if err = encoder.Reconstruct(shards); err != nil { + return fmt.Errorf("recover FEC authentication: %w", err) + } + dataCount := int(f.layout.dataShreds) + data := make(map[uint32]*Shred, len(recovered)) + for _, s := range recovered { + // Chained root and retransmitter signature are outside the erasure region. + // Copy the template suffix now, then replace its proof after authenticating. + copy(s.Payload[shredSignatureSize+f.layout.shardSize:], code.Payload[codingHeaderSize+f.layout.shardSize:]) + data[s.Index-f.fecSetIndex] = s + } + nodes := make([]solana.Hash, 0, merkleTreeSize(len(shards))) + for i, shard := range shards { + var s *Shred + if i < dataCount { + s = f.data[uint32(i)] + if s == nil { + s = data[uint32(i)] + } + } else { + pos := uint16(i - dataCount) + s = f.coding[pos] + if s == nil { + payload := append([]byte(nil), code.Payload...) + binary.LittleEndian.PutUint32(payload[shredIndexOffset:], codeBase+uint32(pos)) + binary.LittleEndian.PutUint16(payload[codingPositionOffset:], pos) + copy(payload[codingHeaderSize:], shard) + // Only header fields and bytes consumed by merkleLeaf are needed here. + copyOfCode := *code + copyOfCode.Payload = payload + copyOfCode.Index = codeBase + uint32(pos) + copyOfCode.Position = pos + s = ©OfCode + } + } + if s == nil { + return fmt.Errorf("recover FEC: missing reconstructed data %d", i) + } + leaf, err := s.merkleLeaf() + if err != nil { + return err + } + nodes = append(nodes, leaf) + } + for size := len(shards); size > 1; size = (size + 1) >> 1 { + offset := len(nodes) - size + for i := 0; i < size; i += 2 { + right := min(i+1, size-1) + nodes = append(nodes, merkleHashNode(nodes[offset+i][:merkleProofEntrySize], nodes[offset+right][:merkleProofEntrySize])) + } + } + if nodes[len(nodes)-1] != expected { + return fmt.Errorf("%w: recovered FEC Merkle root mismatch slot=%d fec_set=%d", ErrInvalidSignature, f.slot, f.fecSetIndex) + } + proofOffset := shredSignatureSize + f.layout.shardSize + if chained { + proofOffset += merkleRootSize + } + for _, s := range recovered { + writeMerkleProof(s.Payload[proofOffset:], nodes, int(s.Index-f.fecSetIndex), len(shards)) + } + return nil +} diff --git a/pkg/turbine/recovery_authentication_test.go b/pkg/turbine/recovery_authentication_test.go new file mode 100644 index 000000000..626742850 --- /dev/null +++ b/pkg/turbine/recovery_authentication_test.go @@ -0,0 +1,226 @@ +package turbine + +import ( + "crypto/ed25519" + "encoding/binary" + "fmt" + "sort" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/klauspost/reedsolomon" + "github.com/stretchr/testify/require" +) + +func resignRecoveryFixture(t *testing.T, packets [][]byte) { + t.Helper() + nodes, err := buildMerkleTree(packets) + require.NoError(t, err) + root := nodes[len(nodes)-1] + sig := ed25519.Sign(ed25519.PrivateKey(benchmarkLeaderKey()), root[:]) + for i, p := range packets { + s, err := ParseShred(p) + require.NoError(t, err) + shard, err := s.erasureShard() + require.NoError(t, err) + start := shredSignatureSize + if s.Type == ShredTypeCode { + start = codingHeaderSize + } + _, chained, _, _ := merkleVariantInfo(s.Variant) + offset := start + len(shard) + if chained { + offset += merkleRootSize + } + copy(p[:shredSignatureSize], sig) + writeMerkleProof(p[offset:], nodes, i, len(packets)) + } +} + +func recoveryFixtureState(t testing.TB, packets [][]byte, missing int) (*SlotAssembler, *slotState) { + t.Helper() + state := &slotState{slot: 10, shreds: make(map[uint32]*Shred), fecSets: make(map[uint32]*fecState), lastIndex: ^uint32(0)} + // Exactly 32 received shreds: also force reconstruction of missing parity. + for i, p := range packets { + if i < missing || i >= 32+missing { + continue + } + s, err := ParseShred(p) + require.NoError(t, err) + state.slot = s.Slot + state.shredVer = s.Version + if s.Type == ShredTypeData { + require.NoError(t, state.addDataShred(s)) + } else { + require.NoError(t, state.addCodingShred(s)) + } + } + return NewSlotAssembler(), state +} + +func TestRecoveredFECAuthentication(t *testing.T) { + for _, resigned := range []bool{false, true} { + for _, missing := range []int{1, 3, 32} { + for _, attack := range []string{"valid", "received_parity", "missing_parity", "recovered_bytes"} { + t.Run(fmt.Sprintf("resigned=%t/missing=%d/%s", resigned, missing, attack), func(t *testing.T) { + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1} + packets, _, _, _, err := gen.MakeShredsFromData(benchmarkLeaderKey(), benchmarkPayload(128), resigned, solana.Hash{9}, 0, 0) + require.NoError(t, err) + require.Len(t, packets, 64) + if attack == "received_parity" { + packets[32][codingHeaderSize+128] ^= 1 + resignRecoveryFixture(t, packets) + } + if attack == "missing_parity" { + if missing == 32 { + t.Skip("no missing coding shard") + } + packets[63][codingHeaderSize+128] ^= 1 + resignRecoveryFixture(t, packets) + } + // The adversarial fixtures have valid leader signatures, not corrupted + // network packets: only RS/tree consistency is wrong. + for _, p := range packets { + s, e := ParseShred(p) + require.NoError(t, e) + require.NoError(t, s.VerifySignature(benchmarkLeaderKey().PublicKey())) + } + a, state := recoveryFixtureState(t, packets, missing) + recovered, err := a.recoverFEC(state, 0) + if attack == "received_parity" || attack == "missing_parity" { + require.ErrorIs(t, err, ErrInvalidSignature) + require.Empty(t, recovered) + return + } + require.NoError(t, err) + require.Len(t, recovered, missing) + for _, s := range recovered { + require.Equal(t, packets[s.Index], s.Payload) + require.NoError(t, s.VerifySignature(benchmarkLeaderKey().PublicKey())) + } + if attack == "recovered_bytes" { + f := state.fecSets[0] + shards := make([][]byte, 64) + for i, s := range f.data { + shards[i], err = s.erasureShard() + require.NoError(t, err) + } + for i, s := range f.coding { + shards[32+int(i)], err = s.erasureShard() + require.NoError(t, err) + } + for _, s := range recovered { + shards[s.Index], err = s.erasureShard() + require.NoError(t, err) + } + recovered[0].Payload[dataHeaderSize+1] ^= 1 + require.ErrorIs(t, a.authenticateRecoveredFEC(f, shards, recovered), ErrInvalidSignature) + } + }) + } + } + } +} + +// Recreate coding packets from immutable Agave data packets, without signing a +// new root. Equality to the root committed by Agave checks erasure layout, +// parity coefficients, coding headers, chain roots and Merkle construction. +func TestRecoveredFECAgaveSignedCapture(t *testing.T) { + all := agavePaddedSlot1752420Packets(t) + sort.Slice(all, func(i, j int) bool { + return binary.LittleEndian.Uint32(all[i][shredIndexOffset:]) < binary.LittleEndian.Uint32(all[j][shredIndexOffset:]) + }) + for group := 0; group < 4; group++ { + packets := make([][]byte, 64) + shards := make([][]byte, 64) + var first *Shred + for i := 0; i < 32; i++ { + // Captured repair packets may append a four-byte nonce, which is + // transport metadata rather than part of the authenticated shred. + packets[i] = all[group*32+i][:dataPayloadSize] + s, err := ParseShred(packets[i]) + require.NoError(t, err) + if i == 0 { + first = s + } + shards[i], err = s.erasureShard() + require.NoError(t, err) + } + codeVariant, ok := merkleCounterpartVariant(first.Variant, ShredTypeCode) + require.True(t, ok) + for i := 0; i < 32; i++ { + p := make([]byte, codingPayloadSize) + copy(p[:codingNumDataOffset], first.Payload[:codingNumDataOffset]) + p[shredVariantOffset] = codeVariant + binary.LittleEndian.PutUint32(p[shredIndexOffset:], first.FECSetIndex+uint32(i)) + binary.LittleEndian.PutUint16(p[codingNumDataOffset:], 32) + binary.LittleEndian.PutUint16(p[codingNumCodingOffset:], 32) + binary.LittleEndian.PutUint16(p[codingPositionOffset:], uint16(i)) + copy(p[codingHeaderSize+len(shards[0]):], first.Payload[shredSignatureSize+len(shards[0]):]) + packets[32+i] = p + shards[32+i] = p[codingHeaderSize : codingHeaderSize+len(shards[0])] + } + enc, err := reedsolomon.New(32, 32) + require.NoError(t, err) + require.NoError(t, enc.Encode(shards)) + nodes, err := buildMerkleTree(packets) + require.NoError(t, err) + root, err := first.MerkleRoot() + require.NoError(t, err) + require.Equal(t, root, nodes[len(nodes)-1]) + proofSize, chained, _, _ := merkleVariantInfo(codeVariant) + offset := codingHeaderSize + len(shards[0]) + if chained { + offset += merkleRootSize + } + for i := 32; i < 64; i++ { + require.Equal(t, int(proofSize), writeMerkleProof(packets[i][offset:], nodes, i, 64)) + } + for _, missing := range []int{1, 3, 32} { + a, state := recoveryFixtureState(t, packets, missing) + got, err := a.recoverFEC(state, first.FECSetIndex) + require.NoError(t, err) + require.Len(t, got, missing) + for _, s := range got { + end := dataPayloadSize + _, _, resigned, _ := merkleVariantInfo(s.Variant) + if resigned { + // Hop signatures can differ by relay; Agave copies the + // received coding template's signature, not a lost one. + end -= shredSignatureSize + want, err := first.RetransmitterSignature() + require.NoError(t, err) + got, err := s.RetransmitterSignature() + require.NoError(t, err) + require.Equal(t, want, got) + } + require.Equal(t, packets[s.Index-first.FECSetIndex][:end], s.Payload[:end]) + actual, err := s.MerkleRoot() + require.NoError(t, err) + require.Equal(t, root, actual) + } + } + } +} + +func BenchmarkAuthenticatedFECRecovery(b *testing.B) { + for _, missing := range []int{1, 3, 32} { + b.Run(fmt.Sprintf("missing_data=%d", missing), func(b *testing.B) { + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1} + packets, _, _, _, err := gen.MakeShredsFromData(benchmarkLeaderKey(), benchmarkPayload(128), false, solana.Hash{9}, 0, 0) + require.NoError(b, err) + a, state := recoveryFixtureState(b, packets, missing) + _, err = a.recoverFEC(state, 0) + require.NoError(b, err) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + recovered, err := a.recoverFEC(state, 0) + if err != nil { + b.Fatal(err) + } + benchmarkRecoveredShredsSink = recovered + } + }) + } +} diff --git a/pkg/turbine/repairsim/ledger.go b/pkg/turbine/repairsim/ledger.go new file mode 100644 index 000000000..3510fcd6f --- /dev/null +++ b/pkg/turbine/repairsim/ledger.go @@ -0,0 +1,257 @@ +// Package repairsim provides a deterministic, single-process repair harness +// around Mithril's production Turbine shred generator and slot assembler. +// +// The synthetic ledger and network are test infrastructure. Shred parsing, +// Merkle/signature validation, repair selection, Reed-Solomon reconstruction, +// component decoding, transaction verification, and completion accounting are +// production code paths. +package repairsim + +import ( + "crypto/ed25519" + "crypto/sha256" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" +) + +const ( + dataShredsPerFEC = 32 + codeShredsPerFEC = 32 +) + +// LedgerConfig controls deterministic canonical-ledger generation. +type LedgerConfig struct { + StartSlot uint64 `json:"start_slot"` + Slots int `json:"slots"` + FECsPerSlot int `json:"fec_sets_per_slot"` + EntriesPerSlot int `json:"entries_per_slot,omitempty"` + Seed int64 `json:"seed"` + ShredVersion uint16 `json:"shred_version"` + ReferenceTick uint8 `json:"reference_tick"` +} + +// Packet is one canonical wire packet and its parsed routing metadata. +type Packet struct { + Bytes []byte + Slot uint64 + Type turbine.ShredType + Index uint32 + FECSetIndex uint32 + Position uint16 +} + +// FECSet contains the canonical packets for one 32+32 FEC set. +type FECSet struct { + Index uint32 + Data []Packet + Coding []Packet +} + +// Slot is the complete canonical source for one generated slot. +type Slot struct { + Number uint64 + ParentSlot uint64 + Entries []turbine.Entry + FECs []FECSet + Data map[uint32]Packet + Highest uint32 +} + +// Ledger is the complete data held by the in-process repair peer. +type Ledger struct { + Config LedgerConfig + Leader solana.PrivateKey + LeaderPub solana.PublicKey + Slots []Slot + bySlot map[uint64]*Slot +} + +// Slot returns a canonical slot by number. +func (l *Ledger) Slot(number uint64) (*Slot, bool) { + if l == nil { + return nil, false + } + s, ok := l.bySlot[number] + return s, ok +} + +// GenerateLedger creates authentic signed Merkle shreds using the production +// 32+32 generator. Entries contain no transactions: this isolates repair, +// reconstruction, parsing, and storage while still exercising the real block +// component codec and completion path. +func GenerateLedger(cfg LedgerConfig) (*Ledger, error) { + if cfg.Slots <= 0 { + return nil, fmt.Errorf("slots must be positive") + } + if cfg.FECsPerSlot <= 0 { + return nil, fmt.Errorf("FEC sets per slot must be positive") + } + if cfg.StartSlot == 0 { + cfg.StartSlot = 10_000 + } + if cfg.ReferenceTick == 0 { + cfg.ReferenceTick = 63 + } + + leader := deterministicLeader(cfg.Seed) + entryCount := cfg.EntriesPerSlot + if entryCount == 0 { + var err error + entryCount, err = findEntryCount(cfg, leader) + if err != nil { + return nil, err + } + } + + ledger := &Ledger{ + Config: cfg, + Leader: leader, + LeaderPub: leader.PublicKey(), + Slots: make([]Slot, 0, cfg.Slots), + bySlot: make(map[uint64]*Slot, cfg.Slots), + } + ledger.Config.EntriesPerSlot = entryCount + for i := 0; i < cfg.Slots; i++ { + number := cfg.StartSlot + uint64(i) + parent := number - 1 + entries := deterministicEntries(cfg.Seed, number, entryCount) + slot, err := generateSlot(cfg, leader, number, parent, entries) + if err != nil { + return nil, fmt.Errorf("generate slot %d: %w", number, err) + } + if got := len(slot.FECs); got != cfg.FECsPerSlot { + return nil, fmt.Errorf("slot %d produced %d FEC sets, want %d (entries=%d)", number, got, cfg.FECsPerSlot, entryCount) + } + ledger.Slots = append(ledger.Slots, slot) + ledger.bySlot[number] = &ledger.Slots[len(ledger.Slots)-1] + } + return ledger, nil +} + +func deterministicLeader(seed int64) solana.PrivateKey { + var input [16]byte + for i := range input { + input[i] = byte(uint64(seed)>>uint((i%8)*8)) ^ byte(i*29+7) + } + digest := sha256.Sum256(append([]byte("mithril-repair-sim-leader-v1"), input[:]...)) + return solana.PrivateKey(ed25519.NewKeyFromSeed(digest[:])) +} + +func deterministicEntries(seed int64, slot uint64, count int) []turbine.Entry { + entries := make([]turbine.Entry, count) + for i := range entries { + material := fmt.Sprintf("mithril-repair-sim-entry-v1:%d:%d:%d", seed, slot, i) + h := sha256.Sum256([]byte(material)) + entries[i] = turbine.Entry{NumHashes: 1, Hash: solana.Hash(h)} + } + return entries +} + +// findEntryCount uses the generator itself as the capacity oracle. This avoids +// duplicating signed-last-FEC payload constants in the harness. +func findEntryCount(cfg LedgerConfig, leader solana.PrivateKey) (int, error) { + lo, hi := 1, cfg.FECsPerSlot*900 + for lo < hi { + mid := lo + (hi-lo)/2 + entries := deterministicEntries(cfg.Seed, cfg.StartSlot, mid) + slot, err := generateSlot(cfg, leader, cfg.StartSlot, cfg.StartSlot-1, entries) + if err != nil { + return 0, err + } + if len(slot.FECs) < cfg.FECsPerSlot { + lo = mid + 1 + } else { + hi = mid + } + } + entries := deterministicEntries(cfg.Seed, cfg.StartSlot, lo) + slot, err := generateSlot(cfg, leader, cfg.StartSlot, cfg.StartSlot-1, entries) + if err != nil { + return 0, err + } + if len(slot.FECs) != cfg.FECsPerSlot { + return 0, fmt.Errorf("cannot derive %d FEC sets within %d entries (got %d)", cfg.FECsPerSlot, hi, len(slot.FECs)) + } + return lo, nil +} + +func generateSlot(cfg LedgerConfig, leader solana.PrivateKey, number, parent uint64, entries []turbine.Entry) (Slot, error) { + component, err := turbine.NewEntryBatch(entries) + if err != nil { + return Slot{}, err + } + shredder := turbine.Shredder{ + Slot: number, + ParentSlot: parent, + Version: cfg.ShredVersion, + ReferenceTick: cfg.ReferenceTick, + } + batch, _, _, err := shredder.MakeMerkleShredsFromComponent( + leader, component, true, solana.Hash{}, 0, 0, + ) + if err != nil { + return Slot{}, err + } + + byFEC := make(map[uint32]*FECSet) + packetByKey := make(map[packetKey][]byte, len(batch.Packets)) + for _, raw := range batch.Packets { + shred, err := turbine.ParseShred(raw) + if err != nil { + return Slot{}, err + } + key := keyForShred(shred) + packetByKey[key] = append([]byte(nil), raw...) + } + data := make(map[uint32]Packet, len(batch.DataShreds)) + var highest uint32 + for _, shred := range append(append([]*turbine.Shred(nil), batch.DataShreds...), batch.CodeShreds...) { + fec := byFEC[shred.FECSetIndex] + if fec == nil { + fec = &FECSet{Index: shred.FECSetIndex} + byFEC[shred.FECSetIndex] = fec + } + packet := Packet{ + Bytes: packetByKey[keyForShred(shred)], + Slot: shred.Slot, + Type: shred.Type, + Index: shred.Index, + FECSetIndex: shred.FECSetIndex, + Position: shred.Position, + } + if shred.Type == turbine.ShredTypeData { + fec.Data = append(fec.Data, packet) + data[shred.Index] = packet + if shred.Index > highest { + highest = shred.Index + } + } else { + fec.Coding = append(fec.Coding, packet) + } + } + fecs := make([]FECSet, 0, len(byFEC)) + for index := uint32(0); len(fecs) < len(byFEC); index += dataShredsPerFEC { + fec := byFEC[index] + if fec == nil { + return Slot{}, fmt.Errorf("non-contiguous FEC sets: missing %d", index) + } + if len(fec.Data) != dataShredsPerFEC || len(fec.Coding) != codeShredsPerFEC { + return Slot{}, fmt.Errorf("FEC %d has %d+%d shreds", index, len(fec.Data), len(fec.Coding)) + } + fecs = append(fecs, *fec) + } + return Slot{Number: number, ParentSlot: parent, Entries: entries, FECs: fecs, Data: data, Highest: highest}, nil +} + +type packetKey struct { + type_ turbine.ShredType + index uint32 + fec uint32 + position uint16 +} + +func keyForShred(shred *turbine.Shred) packetKey { + return packetKey{type_: shred.Type, index: shred.Index, fec: shred.FECSetIndex, position: shred.Position} +} diff --git a/pkg/turbine/repairsim/sim.go b/pkg/turbine/repairsim/sim.go new file mode 100644 index 000000000..c89e867e1 --- /dev/null +++ b/pkg/turbine/repairsim/sim.go @@ -0,0 +1,713 @@ +package repairsim + +import ( + "container/heap" + "errors" + "fmt" + "math/rand" + "os" + "runtime" + "sort" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/turbine" +) + +type Scenario string + +const ( + ScenarioNearTip Scenario = "near-tip" + ScenarioDeepCatchup Scenario = "deep-catchup" +) + +type Availability string + +const ( + AvailabilityComplete Availability = "complete" + AvailabilityNearLoss Availability = "near-loss" + AvailabilitySparse Availability = "sparse" + AvailabilityMixed Availability = "mixed" +) + +// Config controls the deterministic network and local starting state. +type Config struct { + Scenario Scenario `json:"scenario"` + Availability Availability `json:"availability"` + RepairEnabled bool `json:"repair_enabled"` + RepairLatency time.Duration `json:"repair_latency_ns"` + RepairJitter time.Duration `json:"repair_jitter_ns"` + PacketLoss float64 `json:"packet_loss"` + DuplicateProbability float64 `json:"duplicate_probability"` + BandwidthBytesPerSec int64 `json:"bandwidth_bytes_per_sec"` + MaxConcurrent int `json:"max_concurrent_requests"` + MaxRequestSlots int `json:"max_request_slots"` + MaxMissingPerSlot int `json:"max_missing_per_slot"` + CorruptResponses int `json:"corrupt_responses"` + NaturalLateShreds bool `json:"natural_late_shreds"` + CollectTrace bool `json:"collect_trace"` + Seed int64 `json:"seed"` + SpoolDir string `json:"spool_dir,omitempty"` + SpoolMaxBytes int64 `json:"spool_max_bytes"` +} + +// DefaultConfig returns a deterministic starting point for a scenario. +func DefaultConfig(scenario Scenario) Config { + cfg := Config{ + Scenario: scenario, + RepairEnabled: true, + RepairLatency: 20 * time.Millisecond, + RepairJitter: 2 * time.Millisecond, + DuplicateProbability: 0.02, + BandwidthBytesPerSec: 100 * 1024 * 1024, + MaxConcurrent: 256, + MaxRequestSlots: 64, + MaxMissingPerSlot: 256, + NaturalLateShreds: scenario == ScenarioNearTip, + CollectTrace: true, + Seed: 1, + SpoolMaxBytes: 1 << 30, + } + if scenario == ScenarioDeepCatchup { + cfg.Availability = AvailabilityMixed + } else { + cfg.Availability = AvailabilityNearLoss + } + return cfg +} + +// TraceEvent is a deterministic logical-time event. CPU durations are kept in +// Result.StageCPU so trace equality does not depend on scheduler noise. +type TraceEvent struct { + Sequence int `json:"sequence"` + AtNanos int64 `json:"at_ns"` + Stage string `json:"stage"` + Slot uint64 `json:"slot,omitempty"` + FECSetIndex uint32 `json:"fec_set_index,omitempty"` + ShredIndex uint32 `json:"shred_index,omitempty"` + ShredType turbine.ShredType `json:"shred_type,omitempty"` + Bytes int `json:"bytes,omitempty"` + Detail string `json:"detail,omitempty"` +} + +type LatencySummary struct { + P50 time.Duration `json:"p50_ns"` + P95 time.Duration `json:"p95_ns"` + P99 time.Duration `json:"p99_ns"` +} + +// Result separates simulated-network time from actual local execution time. +type Result struct { + Scenario Scenario `json:"scenario"` + Availability Availability `json:"availability"` + Slots int `json:"slots"` + CompletedSlots int `json:"completed_slots"` + LogicalElapsed time.Duration `json:"logical_elapsed_ns"` + WallElapsed time.Duration `json:"wall_elapsed_ns"` + TimeToFirstReplayable time.Duration `json:"time_to_first_replayable_ns"` + TimeToFirstRecoveredData time.Duration `json:"time_to_first_recovered_data_ns"` + RepairEligibleToRecovery time.Duration `json:"repair_eligible_to_first_recovery_ns"` + CompletionLatency LatencySummary `json:"completion_latency"` + SlotsPerLogicalSecond float64 `json:"slots_per_logical_second"` + SlotsPerCPUSecond float64 `json:"slots_per_cpu_second"` + RepairRequests uint64 `json:"repair_requests"` + RepairResponses uint64 `json:"repair_responses"` + RepairBytesRequested uint64 `json:"repair_bytes_requested"` + RepairBytesReceived uint64 `json:"repair_bytes_received"` + UsefulNetworkDataShreds uint64 `json:"useful_network_data_shreds"` + LocallyRecoveredDataShreds uint64 `json:"locally_recovered_data_shreds"` + LocallyRecoveredDataBytes uint64 `json:"locally_recovered_data_bytes"` + InitialMissingDataShreds uint64 `json:"initial_missing_data_shreds"` + FractionRecoveredLocally float64 `json:"fraction_missing_recovered_locally"` + FECDecodes uint64 `json:"fec_decodes"` + DuplicateResponses uint64 `json:"duplicate_responses"` + CanceledOrLateResponses uint64 `json:"canceled_or_late_responses"` + LostResponses uint64 `json:"lost_responses"` + RejectedCorruptResponses uint64 `json:"rejected_corrupt_responses"` + ShredSignatureCacheHits uint64 `json:"shred_signature_cache_hits"` + ShredEd25519Verifications uint64 `json:"shred_ed25519_verifications"` + QueueHighWater int `json:"queue_high_water"` + SpoolBytes int64 `json:"spool_bytes"` + SpoolCompleteSlots int `json:"spool_complete_slots"` + DataShredBytesReplayable uint64 `json:"data_shred_bytes_replayable"` + Allocations uint64 `json:"allocations"` + StageCPU map[string]time.Duration `json:"stage_cpu_ns"` + Trace []TraceEvent `json:"trace"` + Limitations []string `json:"limitations"` +} + +// Run executes the virtual network around production parsing, validation, +// repair selection, reconstruction, spool insertion, and block completion. +func Run(ledger *Ledger, cfg Config) (Result, error) { + if ledger == nil || len(ledger.Slots) == 0 { + return Result{}, errors.New("empty ledger") + } + if cfg.Scenario != ScenarioNearTip && cfg.Scenario != ScenarioDeepCatchup { + return Result{}, fmt.Errorf("unsupported scenario %q", cfg.Scenario) + } + if cfg.Availability == "" { + cfg.Availability = DefaultConfig(cfg.Scenario).Availability + } + if cfg.MaxConcurrent <= 0 { + cfg.MaxConcurrent = 1 + } + if cfg.MaxRequestSlots <= 0 { + cfg.MaxRequestSlots = 64 + } + if cfg.MaxMissingPerSlot <= 0 { + cfg.MaxMissingPerSlot = 256 + } + if cfg.SpoolMaxBytes <= 0 { + cfg.SpoolMaxBytes = 1 << 30 + } + + spoolDir := cfg.SpoolDir + if spoolDir == "" { + var err error + spoolDir, err = os.MkdirTemp("", "mithril-repair-sim-") + if err != nil { + return Result{}, err + } + defer os.RemoveAll(spoolDir) + } + spool, err := turbine.OpenShredSpool(spoolDir, cfg.SpoolMaxBytes) + if err != nil { + return Result{}, err + } + defer spool.Close() + + var memBefore runtime.MemStats + runtime.ReadMemStats(&memBefore) + wallStarted := time.Now() + s := &simulation{ + ledger: ledger, + cfg: cfg, + assembler: turbine.NewSlotAssembler(), + spool: spool, + rng: rand.New(rand.NewSource(cfg.Seed)), + firstShred: make(map[uint64]time.Duration), + completed: make(map[uint64]*block.Block), + replayableAt: make(map[uint64]time.Duration), + pending: make(map[repairKey]struct{}), + stageCPU: make(map[string]time.Duration), + } + s.assembler.SetRetentionFloor(ledger.Slots[0].Number) + s.assembler.SetOnComplete(spool.MarkComplete) + s.nextReplaySlot = ledger.Slots[0].Number + + if err := s.seedLocalState(); err != nil { + return Result{}, err + } + if err := s.runDeliveries(); err != nil { + return Result{}, err + } + + var memAfter runtime.MemStats + runtime.ReadMemStats(&memAfter) + _, spoolBytes := spool.Stats() + result := s.result + result.Scenario = cfg.Scenario + result.Availability = cfg.Availability + result.Slots = len(ledger.Slots) + result.CompletedSlots = len(s.completed) + result.LogicalElapsed = s.now + result.WallElapsed = time.Since(wallStarted) + result.LocallyRecoveredDataShreds = s.assembler.RecoveredDataShreds() + result.InitialMissingDataShreds = initialMissingDataShreds(ledger, cfg.Availability) + if result.InitialMissingDataShreds > 0 { + result.FractionRecoveredLocally = float64(result.LocallyRecoveredDataShreds) / float64(result.InitialMissingDataShreds) + } + if len(ledger.Slots[0].FECs) > 0 && len(ledger.Slots[0].FECs[0].Data) > 0 { + result.LocallyRecoveredDataBytes = result.LocallyRecoveredDataShreds * uint64(len(ledger.Slots[0].FECs[0].Data[0].Bytes)) + } + if s.haveRecoverAt { + result.TimeToFirstRecoveredData = s.firstRecoverAt + if s.haveRepairAt && s.firstRecoverAt >= s.firstRepairAt { + result.RepairEligibleToRecovery = s.firstRecoverAt - s.firstRepairAt + } + } + result.SpoolBytes = spoolBytes + result.SpoolCompleteSlots = spool.CompleteSlots() + result.Allocations = memAfter.Mallocs - memBefore.Mallocs + result.StageCPU = s.stageCPU + result.Trace = s.trace + result.ShredSignatureCacheHits, result.ShredEd25519Verifications = s.shredVerifier.Stats() + result.Limitations = []string{ + "remote peers and latency are simulated in process; no UDP/IP stack is measured", + "synthetic entries contain no transactions, so transaction execution is not measured", + "slot offered to replay means SlotAssembler emitted a verified block; replay execution is not run", + "FEC decode start is observed at the AddShredFrom call boundary, not inside the Reed-Solomon library", + } + latencies := make([]time.Duration, 0, len(s.replayableAt)) + for slot, completedAt := range s.replayableAt { + if first, ok := s.firstShred[slot]; ok { + latencies = append(latencies, completedAt-first) + } + } + result.CompletionLatency = summarizeLatencies(latencies) + if len(s.replayableAt) > 0 { + first := ledger.Slots[0].Number + result.TimeToFirstReplayable = s.replayableAt[first] + } + if result.LogicalElapsed > 0 { + result.SlotsPerLogicalSecond = float64(result.CompletedSlots) / result.LogicalElapsed.Seconds() + } + if result.WallElapsed > 0 { + result.SlotsPerCPUSecond = float64(result.CompletedSlots) / result.WallElapsed.Seconds() + } + return result, nil +} + +type simulation struct { + ledger *Ledger + cfg Config + assembler *turbine.SlotAssembler + shredVerifier turbine.ShredSignatureVerifier + spool *turbine.ShredSpool + rng *rand.Rand + now time.Duration + nextWireAt time.Duration + sequence int + trace []TraceEvent + queue deliveryHeap + pending map[repairKey]struct{} + firstShred map[uint64]time.Duration + completed map[uint64]*block.Block + replayableAt map[uint64]time.Duration + nextReplaySlot uint64 + stageCPU map[string]time.Duration + result Result + corruptLeft int + firstRepairAt time.Duration + haveRepairAt bool + firstRecoverAt time.Duration + haveRecoverAt bool +} + +func (s *simulation) seedLocalState() error { + s.corruptLeft = s.cfg.CorruptResponses + packetsBySlot := make([][]Packet, len(s.ledger.Slots)) + for slotIdx := range s.ledger.Slots { + packetsBySlot[slotIdx] = initialPackets(&s.ledger.Slots[slotIdx], s.cfg.Availability) + } + for ordinal := 0; ; ordinal++ { + added := false + for slotIdx := range s.ledger.Slots { + packets := packetsBySlot[slotIdx] + if ordinal >= len(packets) { + continue + } + added = true + if err := s.ingest(packets[ordinal], false, false); err != nil { + return fmt.Errorf("seed slot %d: %w", s.ledger.Slots[slotIdx].Number, err) + } + s.now++ + } + if !added { + break + } + } + if s.cfg.NaturalLateShreds && s.cfg.Availability == AvailabilityNearLoss { + for i := range s.ledger.Slots { + slot := &s.ledger.Slots[i] + if len(slot.FECs) == 0 || len(slot.FECs[0].Data) < 31 { + continue + } + at := s.now + s.cfg.RepairLatency/2 + time.Duration(i)*time.Microsecond + heap.Push(&s.queue, delivery{at: at, sequence: s.sequence, packet: slot.FECs[0].Data[30]}) + s.sequence++ + } + } + return nil +} + +func initialPackets(slot *Slot, availability Availability) []Packet { + var out []Packet + for i := range slot.FECs { + fec := &slot.FECs[i] + switch availability { + case AvailabilityComplete: + out = append(out, fec.Data...) + case AvailabilityNearLoss: + if i%2 == 0 { + out = append(out, fec.Data[:30]...) + out = append(out, fec.Coding[0]) + } else { + out = append(out, fec.Data[:31]...) + } + case AvailabilitySparse: + out = append(out, fec.Data[:2]...) + case AvailabilityMixed: + out = append(out, fec.Data[:16]...) + out = append(out, fec.Coding[:15]...) + } + } + return out +} + +func (s *simulation) runDeliveries() error { + const maxIterations = 10_000_000 + for iterations := 0; len(s.completed) < len(s.ledger.Slots); iterations++ { + if iterations >= maxIterations { + return errors.New("repair simulation exceeded iteration limit") + } + if s.cfg.RepairEnabled { + s.prioritizeHeadWindow() + s.scheduleRequests() + } + if len(s.queue) == 0 { + // Without repair, exhaust the live arrivals and report any remaining + // holes. No more traffic is expected to complete those slots. + if !s.cfg.RepairEnabled { + return nil + } + return fmt.Errorf("repair stalled with %d/%d completed", len(s.completed), len(s.ledger.Slots)) + } + event := heap.Pop(&s.queue).(delivery) + if event.at > s.now { + s.now = event.at + } + if event.primary { + delete(s.pending, event.key) + } + if event.drop { + s.result.LostResponses++ + s.record("repair_response_lost", event.packet, "") + continue + } + if event.duplicate { + s.result.DuplicateResponses++ + } + if event.fromRepair { + s.result.RepairResponses++ + s.result.RepairBytesReceived += uint64(len(event.packet.Bytes)) + } + beforeUseful := s.assembler.UsefulRepairShreds() + if err := s.ingest(event.packet, event.fromRepair, event.corrupt); err != nil { + if event.corrupt && errors.Is(err, turbine.ErrInvalidSignature) { + s.result.RejectedCorruptResponses++ + s.record("repair_response_rejected", event.packet, "invalid signature or Merkle proof") + continue + } + return err + } + if event.fromRepair { + afterUseful := s.assembler.UsefulRepairShreds() + if afterUseful == beforeUseful { + s.result.CanceledOrLateResponses++ + } else { + s.result.UsefulNetworkDataShreds += afterUseful - beforeUseful + } + } + } + return nil +} + +func (s *simulation) prioritizeHeadWindow() { + var head uint64 + found := false + for i := range s.ledger.Slots { + slot := s.ledger.Slots[i].Number + if _, complete := s.completed[slot]; !complete { + head, found = slot, true + break + } + } + if !found { + return + } + end := head + 63 + last := s.ledger.Slots[len(s.ledger.Slots)-1].Number + if end > last { + end = last + } + s.assembler.PrioritizeRepairRange(head, end) +} + +func (s *simulation) scheduleRequests() { + capacity := s.cfg.MaxConcurrent - len(s.pending) + if capacity <= 0 { + return + } + requests := s.assembler.RepairRequests(s.cfg.MaxRequestSlots, s.cfg.MaxMissingPerSlot) + for _, req := range requests { + if !s.haveRepairAt { + s.firstRepairAt = s.now + s.haveRepairAt = true + } + s.recordAt("repair_needed_decision", req.Slot, 0, 0, 0, 0, + fmt.Sprintf("missing=%d need_highest=%t", len(req.MissingDataShreds), req.NeedHighestDataShred)) + slot, ok := s.ledger.Slot(req.Slot) + if !ok { + continue + } + indexes := append([]uint32(nil), req.MissingDataShreds...) + if req.NeedHighestDataShred { + indexes = append(indexes, slot.Highest) + } + seen := make(map[uint32]struct{}, len(indexes)) + for _, index := range indexes { + if capacity == 0 { + return + } + if _, duplicate := seen[index]; duplicate { + continue + } + seen[index] = struct{}{} + packet, ok := slot.Data[index] + if !ok { + continue + } + key := repairKey{slot: req.Slot, index: index} + if _, outstanding := s.pending[key]; outstanding { + continue + } + s.pending[key] = struct{}{} + capacity-- + s.result.RepairRequests++ + s.result.RepairBytesRequested += uint64(len(packet.Bytes)) + s.record("repair_request_enqueue", packet, "data shred") + s.record("repair_request_send", packet, "data shred") + s.scheduleResponse(key, packet) + } + } +} + +func (s *simulation) scheduleResponse(key repairKey, packet Packet) { + jitter := time.Duration(0) + if s.cfg.RepairJitter > 0 { + span := int64(s.cfg.RepairJitter)*2 + 1 + jitter = time.Duration(s.rng.Int63n(span)) - s.cfg.RepairJitter + } + at := s.now + s.cfg.RepairLatency + jitter + if at < s.now { + at = s.now + } + if at < s.nextWireAt { + at = s.nextWireAt + } + if s.cfg.BandwidthBytesPerSec > 0 { + wire := time.Duration(float64(len(packet.Bytes)) / float64(s.cfg.BandwidthBytesPerSec) * float64(time.Second)) + if wire < time.Nanosecond { + wire = time.Nanosecond + } + at += wire + s.nextWireAt = at + } + d := delivery{at: at, sequence: s.sequence, packet: packet, key: key, primary: true, fromRepair: true} + s.sequence++ + if s.cfg.PacketLoss > 0 && s.rng.Float64() < s.cfg.PacketLoss { + d.drop = true + } + if s.corruptLeft > 0 { + d.corrupt = true + s.corruptLeft-- + } + heap.Push(&s.queue, d) + if !d.drop && s.cfg.DuplicateProbability > 0 && s.rng.Float64() < s.cfg.DuplicateProbability { + dup := d + dup.at++ + dup.sequence = s.sequence + dup.primary = false + dup.duplicate = true + dup.corrupt = false + s.sequence++ + heap.Push(&s.queue, dup) + } + if len(s.queue) > s.result.QueueHighWater { + s.result.QueueHighWater = len(s.queue) + } +} + +func (s *simulation) ingest(packet Packet, fromRepair, corrupt bool) error { + raw := packet.Bytes + if corrupt { + raw = append([]byte(nil), raw...) + if len(raw) > 200 { + raw[200] ^= 0x80 + } else if len(raw) > 0 { + raw[len(raw)-1] ^= 0x80 + } + } + started := time.Now() + shred, err := turbine.ParseShred(raw) + s.stageCPU["shred_parse"] += time.Since(started) + if err != nil { + return err + } + started = time.Now() + err = s.shredVerifier.Verify(shred, s.ledger.LeaderPub) + s.stageCPU["shred_validation"] += time.Since(started) + if err != nil { + return err + } + s.record("shred_validation", packet, "Merkle proof and leader signature valid") + if _, ok := s.firstShred[shred.Slot]; !ok { + s.firstShred[shred.Slot] = s.now + s.record("first_shred", packet, "") + } + started = time.Now() + spooled := s.spool.AppendShred(shred, raw) + s.stageCPU["blockstore_insert"] += time.Since(started) + if spooled { + s.record("blockstore_insert", packet, "verified shred spool") + } + beforeRecovered := s.assembler.RecoveredDataShreds() + started = time.Now() + blk, err := s.assembler.AddShredFrom(shred, fromRepair) + s.stageCPU["assembler_ingest_and_recovery"] += time.Since(started) + if err != nil { + return err + } + afterRecovered := s.assembler.RecoveredDataShreds() + if afterRecovered > beforeRecovered { + s.result.FECDecodes++ + if !s.haveRecoverAt { + s.firstRecoverAt = s.now + s.haveRecoverAt = true + } + s.record("fec_threshold_reached", packet, "observed at AddShredFrom call boundary") + s.record("fec_decode_complete", packet, fmt.Sprintf("recovered_data=%d", afterRecovered-beforeRecovered)) + } + if fromRepair { + s.record("repair_response_receive", packet, "") + } else { + s.record("live_shred_receive", packet, "") + } + if blk != nil { + canonical, ok := s.ledger.Slot(blk.Slot) + if !ok { + return fmt.Errorf("completed unknown slot %d", blk.Slot) + } + if err := compareBlock(canonical, blk); err != nil { + return err + } + if _, ok := s.spool.IsComplete(blk.Slot); !ok { + return fmt.Errorf("slot %d completed without spool completion record", blk.Slot) + } + s.completed[blk.Slot] = blk + for _, data := range canonical.Data { + s.result.DataShredBytesReplayable += uint64(len(data.Bytes)) + } + s.record("slot_offered_to_replay", packet, "verified block emitted") + s.advanceReplayable() + } + return nil +} + +func (s *simulation) advanceReplayable() { + for { + if _, ok := s.completed[s.nextReplaySlot]; !ok { + return + } + s.replayableAt[s.nextReplaySlot] = s.now + s.recordAt("slot_replayable", s.nextReplaySlot, 0, 0, 0, 0, "contiguous parent chain available") + s.nextReplaySlot++ + } +} + +func compareBlock(canonical *Slot, got *block.Block) error { + if got.Slot != canonical.Number || got.SourceParentSlot != canonical.ParentSlot { + return fmt.Errorf("block identity got slot=%d parent=%d, want slot=%d parent=%d", got.Slot, got.SourceParentSlot, canonical.Number, canonical.ParentSlot) + } + if len(got.Entries) != len(canonical.Entries) { + return fmt.Errorf("slot %d entries=%d, want %d", got.Slot, len(got.Entries), len(canonical.Entries)) + } + for i := range canonical.Entries { + want := canonical.Entries[i] + entry := got.Entries[i] + if entry.NumHashes != want.NumHashes || string(entry.Hash) != string(want.Hash[:]) || len(entry.Indices) != len(want.Txns) { + return fmt.Errorf("slot %d entry %d differs from canonical ledger", got.Slot, i) + } + } + if !got.TransactionSignaturesVerified() { + return fmt.Errorf("slot %d block was not signature-verified", got.Slot) + } + return nil +} + +func (s *simulation) record(stage string, packet Packet, detail string) { + s.recordAt(stage, packet.Slot, packet.FECSetIndex, packet.Index, packet.Type, len(packet.Bytes), detail) +} + +func (s *simulation) recordAt(stage string, slot uint64, fec, index uint32, typ turbine.ShredType, bytes int, detail string) { + if s.cfg.CollectTrace { + s.trace = append(s.trace, TraceEvent{ + Sequence: s.sequence, AtNanos: int64(s.now), Stage: stage, Slot: slot, + FECSetIndex: fec, ShredIndex: index, ShredType: typ, Bytes: bytes, Detail: detail, + }) + } + s.sequence++ +} + +func summarizeLatencies(values []time.Duration) LatencySummary { + if len(values) == 0 { + return LatencySummary{} + } + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + percentile := func(p float64) time.Duration { + index := int(float64(len(values)-1)*p + 0.5) + return values[index] + } + return LatencySummary{P50: percentile(.50), P95: percentile(.95), P99: percentile(.99)} +} + +func initialMissingDataShreds(ledger *Ledger, availability Availability) uint64 { + var heldPerFEC int + switch availability { + case AvailabilityComplete: + heldPerFEC = 32 + case AvailabilityNearLoss: + var missing uint64 + for i := range ledger.Slots { + for fec := range ledger.Slots[i].FECs { + if fec%2 == 0 { + missing += 2 + } else { + missing++ + } + } + } + return missing + case AvailabilitySparse: + heldPerFEC = 2 + case AvailabilityMixed: + heldPerFEC = 16 + } + return uint64(len(ledger.Slots) * ledger.Config.FECsPerSlot * (dataShredsPerFEC - heldPerFEC)) +} + +type repairKey struct { + slot uint64 + index uint32 +} + +type delivery struct { + at time.Duration + sequence int + packet Packet + key repairKey + primary bool + fromRepair bool + drop bool + duplicate bool + corrupt bool +} + +type deliveryHeap []delivery + +func (h deliveryHeap) Len() int { return len(h) } +func (h deliveryHeap) Less(i, j int) bool { + if h[i].at != h[j].at { + return h[i].at < h[j].at + } + return h[i].sequence < h[j].sequence +} +func (h deliveryHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *deliveryHeap) Push(x any) { *h = append(*h, x.(delivery)) } +func (h *deliveryHeap) Pop() any { + old := *h + last := old[len(old)-1] + *h = old[:len(old)-1] + return last +} diff --git a/pkg/turbine/repairsim/sim_test.go b/pkg/turbine/repairsim/sim_test.go new file mode 100644 index 000000000..fc161768f --- /dev/null +++ b/pkg/turbine/repairsim/sim_test.go @@ -0,0 +1,254 @@ +package repairsim + +import ( + "reflect" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/turbine" +) + +func testLedger(t *testing.T, slots, fecs int) *Ledger { + t.Helper() + ledger, err := GenerateLedger(LedgerConfig{ + StartSlot: 20_000, + Slots: slots, + FECsPerSlot: fecs, + Seed: 7, + ShredVersion: 11, + ReferenceTick: 63, + }) + if err != nil { + t.Fatal(err) + } + return ledger +} + +func deterministicConfig(scenario Scenario) Config { + cfg := DefaultConfig(scenario) + cfg.RepairLatency = 10 * time.Millisecond + cfg.RepairJitter = time.Millisecond + cfg.DuplicateProbability = 0 + cfg.BandwidthBytesPerSec = 0 + cfg.MaxConcurrent = 32 + cfg.Seed = 19 + return cfg +} + +func TestGenerateLedgerHasExactFECCountAndValidPackets(t *testing.T) { + ledger := testLedger(t, 2, 3) + if ledger.Config.EntriesPerSlot == 0 { + t.Fatal("entry count was not resolved") + } + for _, slot := range ledger.Slots { + if len(slot.FECs) != 3 { + t.Fatalf("slot %d FECs=%d, want 3", slot.Number, len(slot.FECs)) + } + for _, fec := range slot.FECs { + if len(fec.Data) != 32 || len(fec.Coding) != 32 { + t.Fatalf("slot %d FEC %d shape=%d+%d", slot.Number, fec.Index, len(fec.Data), len(fec.Coding)) + } + for _, packet := range append(append([]Packet(nil), fec.Data...), fec.Coding...) { + shred, err := parseAndVerify(packet, ledger) + if err != nil { + t.Fatalf("slot %d FEC %d: %v", slot.Number, fec.Index, err) + } + if shred.Slot != slot.Number || shred.FECSetIndex != fec.Index { + t.Fatalf("packet routing got slot=%d FEC=%d", shred.Slot, shred.FECSetIndex) + } + } + } + } +} + +func TestNearTipRepairCompletesAndTraceIsDeterministic(t *testing.T) { + ledger := testLedger(t, 3, 2) + cfg := deterministicConfig(ScenarioNearTip) + cfg.NaturalLateShreds = true + + first, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + second, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if first.CompletedSlots != 3 || first.SpoolCompleteSlots != 3 { + t.Fatalf("completed=%d spool=%d, want 3", first.CompletedSlots, first.SpoolCompleteSlots) + } + if first.LocallyRecoveredDataShreds == 0 || first.FECDecodes == 0 { + t.Fatalf("recovered=%d decodes=%d, want both nonzero", first.LocallyRecoveredDataShreds, first.FECDecodes) + } + if first.CanceledOrLateResponses == 0 { + t.Fatal("natural late-shred scenario did not produce canceled/late repair work") + } + if !reflect.DeepEqual(first.Trace, second.Trace) { + t.Fatal("same seed/config produced different logical traces") + } +} + +func TestNearTipWithoutRepairRemainsIncomplete(t *testing.T) { + ledger := testLedger(t, 2, 2) + cfg := deterministicConfig(ScenarioNearTip) + cfg.RepairEnabled = false + cfg.NaturalLateShreds = false + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != 0 || result.SpoolCompleteSlots != 0 { + t.Fatalf("completed=%d spool=%d without repair", result.CompletedSlots, result.SpoolCompleteSlots) + } +} + +func TestNaturalLateShredsWithoutRepair(t *testing.T) { + for _, tc := range []struct { + name string + fecs int + wantCompleted int + }{ + {name: "live-arrivals-complete-slots", fecs: 1, wantCompleted: 2}, + {name: "live-arrivals-leave-other-holes", fecs: 2, wantCompleted: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + ledger := testLedger(t, 2, tc.fecs) + cfg := deterministicConfig(ScenarioNearTip) + cfg.RepairEnabled = false + cfg.NaturalLateShreds = true + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != tc.wantCompleted || result.SpoolCompleteSlots != tc.wantCompleted { + t.Fatalf("completed=%d spool=%d, want %d", result.CompletedSlots, result.SpoolCompleteSlots, tc.wantCompleted) + } + // Each live arrival provides the 32nd shard in its slot's first FEC, + // recovering one missing data shred without any repair response. + if result.LocallyRecoveredDataShreds != 2 { + t.Fatalf("recovered=%d, want 2 after both live arrivals", result.LocallyRecoveredDataShreds) + } + if result.RepairRequests != 0 || result.RepairResponses != 0 || result.RepairBytesRequested != 0 { + t.Fatalf("repair disabled: requests=%d responses=%d bytes requested=%d", result.RepairRequests, result.RepairResponses, result.RepairBytesRequested) + } + if result.LogicalElapsed < cfg.RepairLatency/2+time.Microsecond { + t.Fatalf("elapsed=%v, stopped before the last live arrival", result.LogicalElapsed) + } + }) + } +} + +func TestCompleteDeliveryEstablishesZeroRepairBaseline(t *testing.T) { + ledger := testLedger(t, 2, 2) + cfg := deterministicConfig(ScenarioNearTip) + cfg.Availability = AvailabilityComplete + cfg.RepairEnabled = false + cfg.NaturalLateShreds = false + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != 2 { + t.Fatalf("completed=%d, want 2", result.CompletedSlots) + } + if result.RepairRequests != 0 || result.LocallyRecoveredDataShreds != 0 { + t.Fatalf("baseline requests=%d recovered=%d, want zero", result.RepairRequests, result.LocallyRecoveredDataShreds) + } + if result.ShredEd25519Verifications != 4 || result.ShredSignatureCacheHits != 124 { + t.Fatalf("signature cache verifies=%d hits=%d, want 4/124 for four FEC roots", + result.ShredEd25519Verifications, result.ShredSignatureCacheHits) + } +} + +func TestDeepCatchupMixedUsesThresholdRecovery(t *testing.T) { + ledger := testLedger(t, 8, 2) + cfg := deterministicConfig(ScenarioDeepCatchup) + cfg.Availability = AvailabilityMixed + cfg.NaturalLateShreds = false + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != len(ledger.Slots) { + t.Fatalf("completed=%d, want %d", result.CompletedSlots, len(ledger.Slots)) + } + if result.LocallyRecoveredDataShreds <= result.UsefulNetworkDataShreds { + t.Fatalf("local recovery=%d, network data=%d; mixed threshold scenario should recover most losses locally", result.LocallyRecoveredDataShreds, result.UsefulNetworkDataShreds) + } + if result.RepairRequests == 0 || result.RepairBytesReceived == 0 { + t.Fatal("deep catch-up completed without exercising repair") + } +} + +func TestCorruptRepairResponseRejectedThenRetried(t *testing.T) { + ledger := testLedger(t, 1, 1) + cfg := deterministicConfig(ScenarioDeepCatchup) + cfg.Availability = AvailabilityMixed + cfg.CorruptResponses = 1 + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != 1 { + t.Fatalf("completed=%d, want 1", result.CompletedSlots) + } + if result.RejectedCorruptResponses != 1 { + t.Fatalf("rejected corrupt=%d, want 1", result.RejectedCorruptResponses) + } + if result.RepairRequests < 2 { + t.Fatalf("requests=%d, want retry after corruption", result.RepairRequests) + } +} + +func parseAndVerify(packet Packet, ledger *Ledger) (*turbine.Shred, error) { + shred, err := turbine.ParseShred(packet.Bytes) + if err != nil { + return nil, err + } + if err := shred.VerifySignature(ledger.LeaderPub); err != nil { + return nil, err + } + return shred, nil +} + +func BenchmarkScenarios(b *testing.B) { + ledger, err := GenerateLedger(LedgerConfig{ + StartSlot: 30_000, Slots: 8, FECsPerSlot: 2, Seed: 23, ShredVersion: 1, ReferenceTick: 63, + }) + if err != nil { + b.Fatal(err) + } + tests := []struct { + name string + scenario Scenario + availability Availability + }{ + {name: "near-tip", scenario: ScenarioNearTip, availability: AvailabilityNearLoss}, + {name: "deep-mixed", scenario: ScenarioDeepCatchup, availability: AvailabilityMixed}, + {name: "deep-sparse", scenario: ScenarioDeepCatchup, availability: AvailabilitySparse}, + } + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { + cfg := DefaultConfig(tt.scenario) + cfg.Availability = tt.availability + cfg.RepairLatency = 0 + cfg.RepairJitter = 0 + cfg.DuplicateProbability = 0 + cfg.BandwidthBytesPerSec = 0 + cfg.NaturalLateShreds = false + cfg.CollectTrace = false + b.ReportAllocs() + for i := 0; i < b.N; i++ { + result, err := Run(ledger, cfg) + if err != nil { + b.Fatal(err) + } + if result.CompletedSlots != len(ledger.Slots) { + b.Fatalf("completed=%d", result.CompletedSlots) + } + b.ReportMetric(float64(result.RepairRequests), "repair-requests/op") + b.ReportMetric(float64(result.LocallyRecoveredDataShreds), "recovered-shreds/op") + } + }) + } +} diff --git a/pkg/turbine/retention_sweep_test.go b/pkg/turbine/retention_sweep_test.go new file mode 100644 index 000000000..840640461 --- /dev/null +++ b/pkg/turbine/retention_sweep_test.go @@ -0,0 +1,137 @@ +package turbine + +import ( + "errors" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func sweepForTest(a *SlotAssembler) { + a.mu.Lock() + a.pruneOldSlotsLocked() + a.mu.Unlock() +} + +func TestRetentionSweepFloorMovesWithoutNewShreds(t *testing.T) { + a := NewSlotAssembler() + a.maxObservedSlot = 10000 + a.SetRetentionFloor(1000) + a.slots[1000] = &slotState{slot: 1000} + a.slots[1001] = &slotState{slot: 1001} + a.completedSlots[1001] = struct{}{} + a.SetKnownAlpenglowBlockID(1001, solana.Hash{1}) + a.PrioritizeRepairSlot(1000) + sweepForTest(a) + require.Contains(t, a.slots, uint64(1000)) + a.SetRetentionFloor(1001) + sweepForTest(a) + require.NotContains(t, a.slots, uint64(1000)) + require.NotContains(t, a.priorityRepairSlots, uint64(1000)) + require.Contains(t, a.slots, uint64(1001)) + a.SetRetentionFloor(0) + sweepForTest(a) + require.Empty(t, a.slots) + require.Empty(t, a.completedSlots) + require.Empty(t, a.knownBlockIDs) + + // Lowering the floor must permit new old repair state again. + a.SetRetentionFloor(1000) + a.slotState(1000, 1) + sweepForTest(a) + require.Contains(t, a.slots, uint64(1000)) +} + +func TestRetentionSweepReleasesCompletingProtectionAtFixedEdge(t *testing.T) { + for _, outcome := range []string{"abort", "cancel", "error", "complete", "reset"} { + t.Run(outcome, func(t *testing.T) { + a := NewSlotAssembler() + a.maxObservedSlot = 10000 + s := &slotState{slot: 1000, parentSlot: 999, completing: true, shreds: map[uint32]*Shred{0: {}}} + a.slots[s.slot] = s + a.SetKnownAlpenglowBlockID(999, solana.Hash{1}) + a.SetKnownAlpenglowBlockID(1000, solana.Hash{2}) + a.RejectAlpenglowBlockID(1000, solana.Hash{3}) + sweepForTest(a) + require.Contains(t, a.slots, uint64(1000)) + require.Contains(t, a.knownBlockIDs, uint64(999)) + require.Contains(t, a.rejectedBlockIDs, uint64(1000)) + work := &slotCompletionWork{state: s} + switch outcome { + case "abort": + a.abortCompletion(work) + case "cancel": + _, err := a.finalizeCompletion(work, processedSlotCompletion{canceled: true}) + require.NoError(t, err) + case "error": + _, err := a.finalizeCompletion(work, processedSlotCompletion{err: errors.New("decode failure")}) + require.Error(t, err) + case "complete": + _, err := a.finalizeCompletion(work, processedSlotCompletion{block: &block.Block{Slot: 1000}}) + require.NoError(t, err) + case "reset": + a.ResetSlot(1000) + } + sweepForTest(a) + require.Empty(t, a.slots) + require.Empty(t, a.completedSlots) + require.Empty(t, a.knownBlockIDs) + require.Empty(t, a.rejectedBlockIDs) + require.Empty(t, a.partialShredObs) + }) + } +} + +func TestRetentionSweepOldHintsAddedAtFixedEdge(t *testing.T) { + a := NewSlotAssembler() + a.maxObservedSlot = 10000 + sweepForTest(a) + a.SetKnownAlpenglowBlockID(1000, solana.Hash{1}) + a.RejectAlpenglowBlockID(1001, solana.Hash{2}) + sweepForTest(a) + require.Empty(t, a.knownBlockIDs) + require.Empty(t, a.rejectedBlockIDs) + a.mu.Lock() + a.trackBlockIDLocked(&block.Block{Slot: 1002, HasAlpenglowBlockID: true, AlpenglowBlockID: solana.Hash{3}}) + a.mu.Unlock() + sweepForTest(a) + require.Empty(t, a.knownBlockIDs) +} + +func TestRetentionSweepCapacityAtFixedEdge(t *testing.T) { + a := NewSlotAssembler() + a.maxObservedSlot = 10000 + a.SetRetentionFloor(1000) + a.PrioritizeRepairSlot(1001) + sweepForTest(a) + for i := 0; i < maxRetainedIncompleteSlotCap+2; i++ { + a.slotState(1000+uint64(i), 1) + } + sweepForTest(a) + require.Len(t, a.slots, maxRetainedIncompleteSlotCap) + require.Contains(t, a.slots, uint64(1000)) + require.Contains(t, a.slots, uint64(1001)) + require.NotContains(t, a.slots, uint64(1000+maxRetainedIncompleteSlotCap+1)) +} + +func BenchmarkRetentionRepeatedCompletedShred(b *testing.B) { + a := NewSlotAssembler() + a.maxObservedSlot = 10000 + for slot := uint64(9488); slot <= 10000; slot++ { + a.completedSlots[slot] = struct{}{} + a.knownBlockIDs[slot] = solana.Hash{1} + a.rejectedBlockIDs[slot] = map[solana.Hash]struct{}{{2}: {}} + a.partialShredObs[slot] = PartialShredObservation{DataShreds: 1} + } + sh := &Shred{Slot: 10000, Type: ShredTypeData} + // The public ingestion path still acquires the lock and performs its + // ordinary completed-slot rejection on every packet. + _, _ = a.AddShred(sh) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = a.AddShred(sh) + } +} diff --git a/pkg/turbine/retransmit.go b/pkg/turbine/retransmit.go index d8584e6cf..dfb0a04a2 100644 --- a/pkg/turbine/retransmit.go +++ b/pkg/turbine/retransmit.go @@ -60,10 +60,17 @@ type RetransmitConfig struct { type retransmitWork struct { packet []byte - shred ShredID - leader solana.PublicKey + // storage is exclusively owned by this work until send (including retries) + // completes. Nil denotes the unpooled oversized compatibility path. + storage *retransmitPacket + shred ShredID + leader solana.PublicKey } +// Canonical shreds fit in one Solana packet. Keep the pool fixed-size rather +// than retaining arbitrary caller-provided capacities. +type retransmitPacket [packetDataSize]byte + type cachedRetransmitNodes struct { asof time.Time nodes *ClusterNodes @@ -91,6 +98,8 @@ type retransmitParentSigCache struct { } type packetBatchSender interface { + // Send borrows packet and peers only until it returns, including on errors + // or partial sends. Implementations must copy anything they retain. Send(packet []byte, peers []*net.UDPAddr) (int, error) Close() error } @@ -120,8 +129,13 @@ type Retransmitter struct { // immediately visible alongside short-send counters. sendBufferBytes int - queue chan retransmitWork - senders []packetBatchSender + queue chan retransmitWork + senders []packetBatchSender + packetPool sync.Pool + // Only guards queue admission versus shutdown; never held during crypto, + // peer selection or socket writes. A stopped queue cannot retain late work. + submitMu sync.RWMutex + stopped bool cacheMu sync.Mutex cache map[uint64]cachedRetransmitNodes @@ -283,20 +297,53 @@ func (r *Retransmitter) Run(ctx context.Context) { sender := sender go func() { defer workers.Done() + var peers [dataPlaneFanout]*net.UDPAddr for { select { case <-ctx.Done(): return case work := <-r.queue: - r.send(work, sender) + r.send(work, sender, peers[:0]) } } }() } <-ctx.Done() + r.submitMu.Lock() + r.stopped = true + r.submitMu.Unlock() workers.Wait() - for _, sender := range r.senders { - _ = sender.Close() + // Admission is closed and senders have returned their leases. Drain the + // remaining queue without closing a channel still visible to submitters. + for { + select { + case work := <-r.queue: + r.releasePacket(work.storage) + default: + for _, sender := range r.senders { + _ = sender.Close() + } + return + } + } +} + +func (r *Retransmitter) copyPacket(packet []byte) ([]byte, *retransmitPacket) { + if len(packet) > len(retransmitPacket{}) { + return append([]byte(nil), packet...), nil + } + storage, _ := r.packetPool.Get().(*retransmitPacket) + if storage == nil { + storage = new(retransmitPacket) + } + out := storage[:len(packet)] + copy(out, packet) + return out, storage +} + +func (r *Retransmitter) releasePacket(storage *retransmitPacket) { + if storage != nil { + r.packetPool.Put(storage) } } @@ -340,25 +387,37 @@ func (r *Retransmitter) SubmitFrom(packet []byte, shred *Shred, leader solana.Pu if packetSize == 0 || packetSize > len(packet) { return fmt.Errorf("turbine retransmit: invalid canonical packet size %d/%d", packetSize, len(packet)) } - out := append([]byte(nil), packet[:packetSize]...) + // Validate the signing range before borrowing storage, so all error exits + // either precede ownership or release it through send/the queue-drop path. + offset := 0 if root != nil { - offset, err := shred.retransmitterSignatureOffset() + offset, err = shred.retransmitterSignatureOffset() if err != nil { return fmt.Errorf("turbine retransmit: locate retransmitter signature: %w", err) } - if offset+ed25519.SignatureSize > len(out) { - return fmt.Errorf("turbine retransmit: retransmitter signature slice %d:%d exceeds packet size %d", offset, offset+ed25519.SignatureSize, len(out)) + if offset+ed25519.SignatureSize > packetSize { + return fmt.Errorf("turbine retransmit: retransmitter signature slice %d:%d exceeds packet size %d", offset, offset+ed25519.SignatureSize, packetSize) } + } + out, storage := r.copyPacket(packet[:packetSize]) + if root != nil { signature := ed25519.Sign(r.cfg.Identity, root[:]) copy(out[offset:offset+ed25519.SignatureSize], signature) r.resignedShreds.Add(1) } r.submitted.Add(1) + r.submitMu.RLock() + defer r.submitMu.RUnlock() + if r.stopped { + r.releasePacket(storage) + return nil + } select { - case r.queue <- retransmitWork{packet: out, shred: id, leader: leader}: + case r.queue <- retransmitWork{packet: out, storage: storage, shred: id, leader: leader}: default: r.queueDrops.Add(1) + r.releasePacket(storage) } return nil } @@ -417,9 +476,13 @@ func (r *Retransmitter) verifyParentSignature(shred *Shred, leader solana.Public return &root, nil } -func (r *Retransmitter) send(work retransmitWork, sender packetBatchSender) { +func (r *Retransmitter) send(work retransmitWork, sender packetBatchSender, scratch []*net.UDPAddr) { + defer r.releasePacket(work.storage) + // Clear even the unused tail after selection: worker scratch must not pin + // old snapshot addresses after a topology refresh or a no-peer/error path. + defer clear(scratch[:cap(scratch)]) nodes := r.clusterNodesForSlot(work.shred.Slot) - distance, peers, err := nodes.RetransmitPeers(work.leader, work.shred, dataPlaneFanout) + distance, peers, err := nodes.retransmitPeersInto(work.leader, work.shred, dataPlaneFanout, scratch) if errors.Is(err, ErrRetransmitLoopback) { r.loopbacks.Add(1) return diff --git a/pkg/turbine/retransmit_allocation_bench_test.go b/pkg/turbine/retransmit_allocation_bench_test.go new file mode 100644 index 000000000..3f376652a --- /dev/null +++ b/pkg/turbine/retransmit_allocation_bench_test.go @@ -0,0 +1,89 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "encoding/binary" + "fmt" + "net" + "runtime" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/gossip" + "github.com/gagliardetto/solana-go" +) + +type discardRelaySender struct{} + +func (*discardRelaySender) Send(_ []byte, peers []*net.UDPAddr) (int, error) { return len(peers), nil } +func (*discardRelaySender) Close() error { return nil } + +// Includes Submit's dedupe/copy, channel handoff, routing and sender dispatch. +// The sender performs no syscalls; this isolates relay CPU/allocations rather +// than claiming a network-throughput improvement. Authentication before Submit +// and resigned-shred crypto are covered by functional tests, not this fixture. +func BenchmarkRetransmitPipeline(b *testing.B) { + for _, count := range []int{90, 512} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + nodes, leader, key := relayAllocationNodes(count, true) + r, err := newRetransmitterWithSenders(RetransmitConfig{Identity: key, Peers: &mutableTVUPeers{}, Stakes: func(uint64) map[solana.PublicKey]uint64 { return nil }, QueueDepth: 256}, []packetBatchSender{&discardRelaySender{}}) + if err != nil { + b.Fatal(err) + } + r.cache[10] = cachedRetransmitNodes{asof: time.Now().Add(time.Hour), nodes: nodes} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { r.Run(ctx); close(done) }() + packet := make([]byte, dataPayloadSize) + shred := &Shred{Slot: 10, Type: ShredTypeData, Payload: packet} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + // Exactly one producer; leave capacity before submitting so every + // iteration measures actual forwarding rather than dropped work. + for len(r.queue) == cap(r.queue) { + runtime.Gosched() + } + binary.LittleEndian.PutUint64(packet[:8], uint64(i)) + shred.Index = uint32(i) + if err := r.Submit(packet, shred, leader, false); err != nil { + b.Fatal(err) + } + } + for { + var n uint64 + for i := range r.rootDistance { + n += r.rootDistance[i].Load() + } + if n >= uint64(b.N) { + break + } + runtime.Gosched() + } + cancel() + <-done + b.StopTimer() + if r.queueDrops.Load() != 0 { + b.Fatal("unexpected queue drop") + } + }) + } +} + +func relayAllocationNodes(count int, chacha8 bool) (*ClusterNodes, solana.PublicKey, ed25519.PrivateKey) { + key := ed25519.NewKeyFromSeed(make([]byte, ed25519.SeedSize)) + var self solana.PublicKey + copy(self[:], key.Public().(ed25519.PublicKey)) + leader := solana.PublicKey{255} + peers := make([]gossip.TVUPeer, 0, count) + stakes := map[solana.PublicKey]uint64{self: 100, leader: 50} + for i := 0; i < count; i++ { + var pub solana.PublicKey + binary.LittleEndian.PutUint64(pub[:], uint64(i+1)) + addr := &net.UDPAddr{IP: net.IPv4(127, 1, byte(i/250), byte(i%250+1)), Port: 8001} + peers = append(peers, gossip.TVUPeer{Pubkey: gossip.Pubkey(pub), TVUAddr: addr}) + stakes[pub] = uint64(i % 101) + } + return NewRetransmitClusterNodes(ClusterNodesConfig{Self: self, TVUPeers: peers, Stakes: stakes, UseChaCha8: chacha8}), leader, key +} diff --git a/pkg/turbine/retransmit_allocation_test.go b/pkg/turbine/retransmit_allocation_test.go new file mode 100644 index 000000000..3cced3129 --- /dev/null +++ b/pkg/turbine/retransmit_allocation_test.go @@ -0,0 +1,267 @@ +package turbine + +import ( + "bytes" + "context" + "net" + "sync" + "syscall" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Derive expected children from the full weighted permutation, independently +// of the streaming child-selection loop and its caller-owned output storage. +func expectedRelayPeers(nodes *ClusterNodes, leader solana.PublicKey, id ShredID, fanout int) (uint8, []*net.UDPAddr) { + order := nodes.retransmitShuffle(leader, id) + self := -1 + for i, index := range order { + if nodes.nodes[index].pubkey == nodes.selfPubkey { + self = i + break + } + } + if self < 0 { + return maxTurbineHops - 1, nil + } + offset := 0 + if self > 0 { + offset = (self - 1) % fanout + } + step := fanout + if self == 0 { + step = 1 + } + var peers []*net.UDPAddr + for position, n := (self-offset)*fanout+offset+1, 0; position < len(order) && n < fanout; position, n = position+step, n+1 { + node := nodes.nodes[order[position]] + if node.hasContact { + if addr, ok := broadcastTVUUDP(node.tvuAddr); ok { + peers = append(peers, addr) + } + } + } + return turbineRootDistance(self, fanout), peers +} + +func TestRetransmitScratchMatchesPermutation(t *testing.T) { + for _, chacha8 := range []bool{false, true} { + for _, count := range []int{0, 31, 90, 512} { + nodes, leader, _ := relayAllocationNodes(count, chacha8) + // Exercise absent contacts and unroutable peers without changing stake order. + for i := range nodes.nodes { + if i%13 == 0 { + nodes.nodes[i].hasContact = false + } + if i%17 == 0 { + nodes.nodes[i].tvuAddr = &net.UDPAddr{IP: net.ParseIP("::1"), Port: 8001} + } + } + for _, fanout := range []int{1, 3, 200} { + var scratch [dataPlaneFanout]*net.UDPAddr + for index := uint32(0); index < 40; index++ { + id := ShredID{Slot: uint64(10 + index/4), Index: index, Type: ShredType(index % 2)} + wantDistance, want := expectedRelayPeers(nodes, leader, id, fanout) + distance, got, err := nodes.retransmitPeersInto(leader, id, fanout, scratch[:0]) + require.NoError(t, err) + require.Equal(t, wantDistance, distance) + require.Equal(t, len(want), len(got)) + for i := range want { + require.Same(t, want[i], got[i]) + } + _, owned, err := nodes.RetransmitPeers(leader, id, fanout) + require.NoError(t, err) + copyOfOwned := append([]*net.UDPAddr(nil), owned...) + clear(scratch[:]) + require.Equal(t, copyOfOwned, append([]*net.UDPAddr(nil), owned...)) + } + } + } + } +} + +func TestRetransmitScratchConcurrent(t *testing.T) { + nodes, leader, _ := relayAllocationNodes(300, true) + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + var scratch [dataPlaneFanout]*net.UDPAddr + for index := uint32(0); index < 50; index++ { + id := ShredID{Slot: 10, Index: index, Type: ShredTypeData} + wd, wp := expectedRelayPeers(nodes, leader, id, 3) + d, p, err := nodes.retransmitPeersInto(leader, id, 3, scratch[:0]) + if err != nil || d != wd || len(p) != len(wp) { + t.Error("concurrent routing mismatch") + return + } + for i := range p { + if p[i] != wp[i] { + t.Error("concurrent peer mismatch") + return + } + } + } + }) + } + wg.Wait() +} + +func TestRetransmitPacketPoolOwnership(t *testing.T) { + r := &Retransmitter{} + input := bytes.Repeat([]byte{7}, packetDataSize) + a, ownerA := r.copyPacket(input) + b, ownerB := r.copyPacket(input) + require.NotSame(t, ownerA, ownerB) + clear(input) + require.Equal(t, byte(7), a[0]) + require.Equal(t, byte(7), b[0]) + r.releasePacket(ownerA) + for range 50 { + p, owner := r.copyPacket(bytes.Repeat([]byte{9}, 1203)) + clear(p) + r.releasePacket(owner) + } + require.Equal(t, bytes.Repeat([]byte{7}, packetDataSize), b) + r.releasePacket(ownerB) + oversized := bytes.Repeat([]byte{4}, packetDataSize+1) + p, owner := r.copyPacket(oversized) + require.Nil(t, owner) + clear(oversized) + require.Equal(t, byte(4), p[0]) +} + +func TestRetransmitQueuedPacketOwnsInput(t *testing.T) { + nodes, leader, key := relayAllocationNodes(90, true) + r, err := newRetransmitterWithSenders(RetransmitConfig{Identity: key, Peers: &mutableTVUPeers{}, Stakes: func(uint64) map[solana.PublicKey]uint64 { return nil }, QueueDepth: 1}, []packetBatchSender{newCaptureBatchSender()}) + require.NoError(t, err) + r.cache[10] = cachedRetransmitNodes{asof: time.Now(), nodes: nodes} + packet := bytes.Repeat([]byte{3}, dataPayloadSize) + packet[0] = 1 + shred := &Shred{Slot: 10, Index: 1, Type: ShredTypeData, Payload: packet} + require.NoError(t, r.Submit(packet, shred, leader, false)) + want := append([]byte(nil), packet...) + for i := 2; i < 20; i++ { + packet[0] = byte(i) + shred.Index = uint32(i) + require.NoError(t, r.Submit(packet, shred, leader, false)) + } + require.Equal(t, uint64(18), r.queueDrops.Load()) + clear(packet) + work := <-r.queue + require.Equal(t, want, work.packet) + // Cancellation leaves no owned copies queued after workers finish. + r.queue <- work + ctx, cancel := context.WithCancel(context.Background()) + cancel() + r.Run(ctx) + require.Empty(t, r.queue) + packet[0] = 99 + shred.Index = 99 + require.NoError(t, r.Submit(packet, shred, leader, false)) + require.Empty(t, r.queue, "late submit retained storage after workers stopped") +} + +type borrowedRelaySender struct { + t *testing.T + want []byte + entered chan struct{} + resume chan struct{} + calls int +} + +func (s *borrowedRelaySender) Send(packet []byte, peers []*net.UDPAddr) (int, error) { + s.calls++ + if s.calls == 1 { + close(s.entered) + <-s.resume + } + require.Equal(s.t, s.want, packet) + if s.calls == 1 { + return 0, syscall.EAGAIN + } + return len(peers), nil +} +func (*borrowedRelaySender) Close() error { return nil } + +func TestRetransmitPoolLeaseSurvivesSendRetries(t *testing.T) { + nodes, leader, key := relayAllocationNodes(31, true) + id := ShredID{Slot: 10, Type: ShredTypeData} + // Select a shred for which this validator is the root and has children. + for ; id.Index < 10000; id.Index++ { + d, p, err := nodes.RetransmitPeers(leader, id, 200) + require.NoError(t, err) + if d == 0 && len(p) > 1 { + break + } + } + require.Less(t, id.Index, uint32(10000)) + want := bytes.Repeat([]byte{7}, dataPayloadSize) + sender := &borrowedRelaySender{t: t, want: want, entered: make(chan struct{}), resume: make(chan struct{})} + r, err := newRetransmitterWithSenders(RetransmitConfig{Identity: key, Peers: &mutableTVUPeers{}, Stakes: func(uint64) map[solana.PublicKey]uint64 { return nil }}, []packetBatchSender{sender}) + require.NoError(t, err) + r.cache[10] = cachedRetransmitNodes{asof: time.Now(), nodes: nodes} + packet, owner := r.copyPacket(want) + var scratch [dataPlaneFanout]*net.UDPAddr + done := make(chan struct{}) + go func() { + defer close(done) + r.send(retransmitWork{packet: packet, storage: owner, shred: id, leader: leader}, sender, scratch[:0]) + }() + select { + case <-sender.entered: + case <-time.After(5 * time.Second): + t.Fatal("send did not start") + } + for range 100 { + p, owned := r.copyPacket(want) + clear(p) + r.releasePacket(owned) + } + close(sender.resume) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("send did not finish") + } + require.Equal(t, 2, sender.calls) + for _, addr := range scratch { + require.Nil(t, addr, "scratch pinned prior snapshot") + } +} + +func TestRetransmitConcurrentSubmitAndStop(t *testing.T) { + nodes, leader, key := relayAllocationNodes(90, true) + r, err := newRetransmitterWithSenders(RetransmitConfig{Identity: key, Peers: &mutableTVUPeers{}, Stakes: func(uint64) map[solana.PublicKey]uint64 { return nil }, QueueDepth: 8}, []packetBatchSender{&discardRelaySender{}, &discardRelaySender{}}) + require.NoError(t, err) + r.cache[10] = cachedRetransmitNodes{asof: time.Now(), nodes: nodes} + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { r.Run(ctx); close(done) }() + var wg sync.WaitGroup + for worker := 0; worker < 8; worker++ { + wg.Go(func() { + packet := make([]byte, dataPayloadSize) + for i := 0; i < 100; i++ { + packet[0], packet[1] = byte(worker), byte(i) + shred := &Shred{Slot: 10, Index: uint32(worker*100 + i), Type: ShredTypeData, Payload: packet} + if err := r.Submit(packet, shred, leader, false); err != nil { + t.Error(err) + } + if i == 50 { + cancel() + } + } + }) + } + wg.Wait() + cancel() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("shutdown blocked") + } + require.Empty(t, r.queue) +} diff --git a/pkg/turbine/shred.go b/pkg/turbine/shred.go index 4361f7cbd..23a4d002a 100644 --- a/pkg/turbine/shred.go +++ b/pkg/turbine/shred.go @@ -457,13 +457,12 @@ func merkleHashNode(left []byte, right []byte) solana.Hash { return hashv([][]byte{[]byte(merkleHashPrefixNode), left, right}) } -func hashv(parts [][]byte) solana.Hash { +func hashv(parts [][]byte) (out solana.Hash) { h := sha256.New() for _, part := range parts { _, _ = h.Write(part) } - var out solana.Hash - copy(out[:], h.Sum(nil)) + _ = h.Sum(out[:0]) return out } diff --git a/pkg/turbine/shredspool.go b/pkg/turbine/shredspool.go index 8f8003eb1..0170b0f76 100644 --- a/pkg/turbine/shredspool.go +++ b/pkg/turbine/shredspool.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "sync" + + "github.com/Overclock-Validator/mithril/pkg/mlog" ) // ShredSpool is a disposable on-disk cache of VERIFIED raw shreds, one @@ -32,25 +34,28 @@ import ( // re-fetch near the tip, the low end borders replay and is what repair would // otherwise pay for dearly. type ShredSpool struct { - mu sync.Mutex - dir string - open map[uint64]*spoolFile - sizes map[uint64]int64 // per-slot bytes on disk (open writers included) - seen map[uint64]map[spoolShredKey]struct{} // distinct shreds appended this run - validated map[uint64]bool // adopted files whose record tail was checked this run - complete map[uint64]SpoolSlotMeta - journal *os.File // append-only completeness journal (complete.idx) - bytes int64 - maxBytes int64 - highestSlot uint64 - haveHighest bool - floor uint64 - closed bool + mu sync.Mutex + dir string + open map[uint64]*spoolFile + sizes map[uint64]int64 // per-slot bytes on disk (open writers included) + seen map[uint64]map[spoolShredKey]struct{} // distinct shreds appended this run + validated map[uint64]bool // adopted files whose record tail was checked this run + complete map[uint64]SpoolSlotMeta + journal *spoolCompletionJournal // ordered completeness hints (complete.idx) + journalOverflow bool // Close must retry the current hints after queue overflow + bytes int64 + maxBytes int64 + highestSlot uint64 + haveHighest bool + floor uint64 + closed bool } // SpoolSlotMeta records a slot proven FULLY assembled: every data shred // 0..LastIndex was held when the assembler completed it. The completeness -// index is what turns the spool from a byte cache into the seed of a +// index is a repair hint, not proof that all buffered packets survived a crash. +// The assembler still validates coverage when hydrating a slot. This turns +// the spool from a byte cache into the seed of a // repair-serving shredstore: complete slots need zero network on restart, // answer HighestWindowIndex honestly, and define the serving/retention set. type SpoolSlotMeta struct { @@ -170,10 +175,10 @@ func (s *ShredSpool) loadJournal() { if err != nil { return // journal unavailable: completeness degrades to per-run only } + s.journal = newSpoolCompletionJournal(f) for slot, meta := range s.complete { - f.Write(spoolJournalRecord(slot, meta)) + s.journal.complete(slot, meta) } - s.journal = f } func spoolJournalRecord(slot uint64, meta SpoolSlotMeta) []byte { @@ -186,11 +191,13 @@ func spoolJournalRecord(slot uint64, meta SpoolSlotMeta) []byte { // MarkComplete records that the slot fully assembled (data shreds // 0..lastIndex all held). Called by the assembler's completion hook, so -// hydrating an adopted file re-marks it for free. Idempotent. +// hydrating an adopted file re-marks it for free. Idempotent. Journal submission +// never waits for storage or queue space; a crash may lose this repair hint, +// causing reassembly/repair, but cannot authorize a vote or advance a checkpoint. func (s *ShredSpool) MarkComplete(slot uint64, lastIndex uint32, shreds uint32) { s.mu.Lock() defer s.mu.Unlock() - if slot < s.floor || shreds == 0 { + if s.closed || slot < s.floor || shreds == 0 { return } if _, done := s.complete[slot]; done { @@ -198,8 +205,8 @@ func (s *ShredSpool) MarkComplete(slot uint64, lastIndex uint32, shreds uint32) } meta := SpoolSlotMeta{LastIndex: lastIndex, Shreds: shreds} s.complete[slot] = meta - if s.journal != nil { - s.journal.Write(spoolJournalRecord(slot, meta)) + if s.journal != nil && !s.journal.tryComplete(slot, meta) { + s.journalOverflow = true } } @@ -371,12 +378,18 @@ func (s *ShredSpool) ensureRoomLocked(slot uint64, additional int64) bool { if slot >= s.highestSlot { return false } - s.dropSlotLocked(s.highestSlot) + if !s.dropSlotLocked(s.highestSlot) { + return false + } } return true } -func (s *ShredSpool) dropSlotLocked(slot uint64) { +func (s *ShredSpool) dropSlotLocked(slot uint64) bool { + if err := s.invalidateCompleteLocked(slot); err != nil { + mlog.Log.Warnf("shred spool: retaining slot %d after completion invalidation failed: %v", slot, err) + return false + } s.closeSlotLocked(slot) s.bytes -= s.sizes[slot] delete(s.sizes, slot) @@ -384,14 +397,20 @@ func (s *ShredSpool) dropSlotLocked(slot uint64) { delete(s.validated, slot) delete(s.complete, slot) _ = os.Remove(s.pathFor(slot)) - if s.journal != nil { - // Supersede any older completion record if this slot number is later - // re-created before the journal is compacted on restart. - s.journal.Write(spoolJournalRecord(slot, SpoolSlotMeta{})) - } if s.haveHighest && slot == s.highestSlot { s.recomputeHighestLocked() } + return true +} + +// Remove the live hint immediately, but do not mutate its file until older +// journal hints have been superseded. A failed fence leaves the file intact. +func (s *ShredSpool) invalidateCompleteLocked(slot uint64) error { + delete(s.complete, slot) + if s.journal != nil { + return s.journal.invalidate(slot) + } + return nil } func (s *ShredSpool) recomputeHighestLocked() { @@ -483,16 +502,15 @@ func (s *ShredSpool) readSlotLocked(slot uint64) ([][]byte, error) { validEnd = packetEnd } if validEnd != len(data) { + if err := s.invalidateCompleteLocked(slot); err != nil { + return nil, err + } if err := os.Truncate(path, int64(validEnd)); err != nil { return nil, fmt.Errorf("truncate corrupt shred spool tail for slot %d: %w", slot, err) } oldSize := s.sizes[slot] s.sizes[slot] = int64(validEnd) s.bytes += int64(validEnd) - oldSize - delete(s.complete, slot) - if s.journal != nil { - s.journal.Write(spoolJournalRecord(slot, SpoolSlotMeta{})) - } } s.validated[slot] = true return packets, nil @@ -533,12 +551,20 @@ func (s *ShredSpool) Stats() (slots int, bytes int64) { func (s *ShredSpool) Close() { s.mu.Lock() defer s.mu.Unlock() + if s.closed { + return + } s.closed = true for slot := range s.open { s.closeSlotLocked(slot) } if s.journal != nil { - _ = s.journal.Close() + if s.journalOverflow { + for slot, meta := range s.complete { + s.journal.complete(slot, meta) + } + } + s.journal.close() s.journal = nil } } diff --git a/pkg/turbine/shredspool_benchmark_test.go b/pkg/turbine/shredspool_benchmark_test.go new file mode 100644 index 000000000..d3e30cf6f --- /dev/null +++ b/pkg/turbine/shredspool_benchmark_test.go @@ -0,0 +1,37 @@ +package turbine + +import ( + "path/filepath" + "sort" + "strconv" + "testing" + "time" +) + +// Run this identical public-API benchmark on both revisions. Each sample owns +// a fresh spool, so no completed-slot dedupe or full-queue dropping is timed. +// Close is untimed but drains the writer before the next sample. These ordinary +// filesystem measurements do not simulate rare storage stalls or whole replay. +func BenchmarkShredSpoolMarkComplete(b *testing.B) { + root := b.TempDir() + elapsed := make([]int64, 0, b.N) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + b.StopTimer() + s, err := OpenShredSpool(filepath.Join(root, strconv.Itoa(i)), 0) + if err != nil { + b.Fatal(err) + } + s.Append(100, []byte("packet")) + b.StartTimer() + start := time.Now() + s.MarkComplete(100, 0, 1) + duration := time.Since(start).Nanoseconds() + b.StopTimer() + elapsed = append(elapsed, duration) + s.Close() + } + sort.Slice(elapsed, func(i, j int) bool { return elapsed[i] < elapsed[j] }) + b.ReportMetric(float64(elapsed[(len(elapsed)-1)/2]), "p50-ns") + b.ReportMetric(float64(elapsed[(99*len(elapsed)+99)/100-1]), "p99-ns") +} diff --git a/pkg/turbine/shredspool_journal.go b/pkg/turbine/shredspool_journal.go new file mode 100644 index 000000000..a537e202f --- /dev/null +++ b/pkg/turbine/shredspool_journal.go @@ -0,0 +1,102 @@ +package turbine + +import ( + "fmt" + "io" +) + +// Completion records are repair-cache hints, not voting or checkpoint state. +// A bounded writer removes their disk I/O from block delivery. Only completion +// hints may be dropped when the queue is full; live completeness stays in memory +// and Close retries the current hints before the next opener takes ownership. +const spoolJournalQueueSize = 256 + +type spoolJournalFile interface { + io.Writer + Truncate(int64) error + Close() error +} + +type spoolJournalRequest struct { + record [spoolJournalRecordSize]byte + done chan error // non-nil for an invalidation that must precede file mutation +} + +type spoolCompletionJournal struct { + requests chan spoolJournalRequest + done chan struct{} +} + +func newSpoolCompletionJournal(file spoolJournalFile) *spoolCompletionJournal { + j := &spoolCompletionJournal{requests: make(chan spoolJournalRequest, spoolJournalQueueSize), done: make(chan struct{})} + go j.run(file) + return j +} + +func journalRequest(slot uint64, meta SpoolSlotMeta) spoolJournalRequest { + var req spoolJournalRequest + copy(req.record[:], spoolJournalRecord(slot, meta)) + return req +} + +// The spool mutex serializes submissions and Close, but the worker never takes +// that mutex. A stalled write therefore cannot directly stall MarkComplete. +func (j *spoolCompletionJournal) tryComplete(slot uint64, meta SpoolSlotMeta) bool { + select { + case j.requests <- journalRequest(slot, meta): + return true + default: + return false + } +} + +func (j *spoolCompletionJournal) complete(slot uint64, meta SpoolSlotMeta) { + j.requests <- journalRequest(slot, meta) +} + +// Never drop or reorder invalidations. Wait for earlier completions and this +// tombstone before deleting/replacing/truncating a slot file. These rare paths +// may still wait for storage while holding the spool mutex. Queueing tombstones +// without this fence could resurrect an old completion after a crash. +func (j *spoolCompletionJournal) invalidate(slot uint64) error { + req := journalRequest(slot, SpoolSlotMeta{}) + req.done = make(chan error, 1) + j.requests <- req + return <-req.done +} + +func (j *spoolCompletionJournal) close() { + close(j.requests) + <-j.done +} + +func (j *spoolCompletionJournal) run(file spoolJournalFile) { + defer close(j.done) + defer file.Close() + failed, invalidated := false, false + for req := range j.requests { + var err error + if !failed { + var n int + n, err = file.Write(req.record[:]) + if err == nil && n != len(req.record) { + err = io.ErrShortWrite + } + failed = err != nil + } + if failed && !invalidated { + // Never append behind a short record, or acknowledge an invalidation + // while old completion hints remain. Empty the disposable journal and + // disable further hint writes for this opener. If even truncation fails, + // the caller must leave the slot file unchanged and retry later. + err = file.Truncate(0) + invalidated = err == nil + if err != nil { + err = fmt.Errorf("invalidate failed shred completeness journal: %w", err) + } + } + if req.done != nil { + req.done <- err + } + } +} diff --git a/pkg/turbine/shredspool_journal_test.go b/pkg/turbine/shredspool_journal_test.go new file mode 100644 index 000000000..1ba9169ce --- /dev/null +++ b/pkg/turbine/shredspool_journal_test.go @@ -0,0 +1,205 @@ +package turbine + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "sync" + "testing" + "time" +) + +type gatedSpoolJournal struct { + *os.File + started chan struct{} + release chan struct{} + once sync.Once +} + +func (f *gatedSpoolJournal) Write(p []byte) (int, error) { + f.once.Do(func() { close(f.started); <-f.release }) + return f.File.Write(p) +} +func gateSpoolJournal(t *testing.T, s *ShredSpool) *gatedSpoolJournal { + t.Helper() + s.journal.close() + file, err := os.OpenFile(filepath.Join(s.dir, spoolJournalName), os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + gate := &gatedSpoolJournal{File: file, started: make(chan struct{}), release: make(chan struct{})} + s.journal = newSpoolCompletionJournal(gate) + return gate +} +func waitSpoolTest(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("spool operation did not complete") + } +} + +func TestShredSpoolCompletionDoesNotWaitForJournalAndCloseDrainsOverflow(t *testing.T) { + dir := t.TempDir() + s, err := OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + // Seed real slot files before blocking only the completeness writer. + for slot := uint64(1); slot <= spoolJournalQueueSize+8; slot++ { + s.Append(slot, []byte("packet")) + } + gate := gateSpoolJournal(t, s) + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(gate.release) }); s.Close() }) + s.MarkComplete(1, 0, 1) + waitSpoolTest(t, gate.started) + done := make(chan struct{}) + go func() { + for slot := uint64(2); slot <= spoolJournalQueueSize+8; slot++ { + s.MarkComplete(slot, 0, 1) + } + close(done) + }() + waitSpoolTest(t, done) + if !s.journalOverflow { + t.Fatal("test did not overflow the bounded queue") + } + if s.CompleteSlots() != spoolJournalQueueSize+8 { + t.Fatal("overflow lost live completeness") + } + closed := make(chan struct{}) + go func() { s.Close(); close(closed) }() + select { + case <-closed: + t.Fatal("Close returned before the blocked writer drained") + default: + } + release.Do(func() { close(gate.release) }) + waitSpoolTest(t, closed) + s.Close() // idempotent; no closed-channel send + s.MarkComplete(9999, 0, 1) + if _, ok := s.IsComplete(9999); ok { + t.Fatal("post-Close completion was accepted") + } + reopened, err := OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if reopened.CompleteSlots() != spoolJournalQueueSize+8 { + t.Fatal("clean handoff lost overflowed completion hints") + } +} + +func TestShredSpoolInvalidationWaitsBeforeReplacement(t *testing.T) { + dir := t.TempDir() + s, err := OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + s.Append(800, []byte("old-file")) + if _, err := s.ReadSlot(800); err != nil { + t.Fatal(err) + } + gate := gateSpoolJournal(t, s) + var release sync.Once + t.Cleanup(func() { release.Do(func() { close(gate.release) }); s.Close() }) + s.MarkComplete(800, 0, 1) + waitSpoolTest(t, gate.started) + done := make(chan struct{}) + go func() { s.DiscardSlot(800); s.Append(800, []byte("replacement-partial")); close(done) }() + // While the earlier completion write is stalled, replacement must wait. + select { + case <-done: + t.Fatal("replacement passed an undrained invalidation") + case <-time.After(20 * time.Millisecond): + } + data, err := os.ReadFile(s.pathFor(800)) + if err != nil || !bytes.Contains(data, []byte("old-file")) { + t.Fatalf("old file mutated before invalidation: %v", err) + } + release.Do(func() { close(gate.release) }) + waitSpoolTest(t, done) + s.Close() + reopened, err := OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if _, ok := reopened.IsComplete(800); ok { + t.Fatal("older completion resurrected for replacement") + } + packets, err := reopened.ReadSlot(800) + if err != nil || len(packets) != 1 || string(packets[0]) != "replacement-partial" { + t.Fatalf("replacement: %q %v", packets, err) + } +} + +type faultySpoolJournal struct { + *os.File + failTruncate bool // only changed while worker is fenced by invalidate's reply +} + +func (f *faultySpoolJournal) Write(p []byte) (int, error) { return f.File.Write(p[:len(p)/2]) } +func (f *faultySpoolJournal) Truncate(n int64) error { + if f.failTruncate { + return errors.New("injected truncate failure") + } + return f.File.Truncate(n) +} + +func TestShredSpoolJournalShortWriteDisablesHintsAndFencesMutations(t *testing.T) { + for _, failTruncate := range []bool{false, true} { + dir := t.TempDir() + s, err := OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + s.Append(100, []byte("old-file")) + s.MarkComplete(100, 0, 1) + s.Close() + s, err = OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + s.journal.close() + file, err := os.OpenFile(filepath.Join(dir, spoolJournalName), os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatal(err) + } + faulty := &faultySpoolJournal{File: file, failTruncate: failTruncate} + s.journal = newSpoolCompletionJournal(faulty) + s.mu.Lock() + dropped := s.dropSlotLocked(100) + s.mu.Unlock() + if dropped == failTruncate { + t.Fatalf("drop=%v with truncate failure=%v", dropped, failTruncate) + } + if failTruncate { + data, err := os.ReadFile(s.pathFor(100)) + if err != nil || !bytes.Contains(data, []byte("old-file")) { + t.Fatal("failed journal fence mutated slot file") + } + faulty.failTruncate = false + s.DiscardSlot(100) // retry can now invalidate all old hints + } + s.Append(100, []byte("partial")) + s.MarkComplete(101, 0, 1) + s.Close() + data, err := os.ReadFile(filepath.Join(dir, spoolJournalName)) + if err != nil || len(data) != 0 { + t.Fatalf("failed journal must stay empty, got %d bytes: %v", len(data), err) + } + reopened, err := OpenShredSpool(dir, 0) + if err != nil { + t.Fatal(err) + } + if _, ok := reopened.IsComplete(100); ok { + t.Fatal("failed journal resurrected completion") + } + reopened.Close() + } +} diff --git a/pkg/turbine/sigcache.go b/pkg/turbine/sigcache.go index 7f3b13139..990023d9b 100644 --- a/pkg/turbine/sigcache.go +++ b/pkg/turbine/sigcache.go @@ -20,7 +20,13 @@ import ( // hit reproduces exactly the result of re-running it on the same inputs. // Tampered content can never hit — different bytes yield a different root, // hence a different key. Failures are never cached. -type shredSigCache struct { +// ShredSignatureVerifier authenticates Merkle shreds with the same bounded, +// per-root result cache used by UDPReceiver. The cache never stores failures; +// each packet's Merkle proof is still evaluated before a cache lookup. +// +// It is exported so deterministic and loopback ingress harnesses can exercise +// production validation without constructing a UDPReceiver. +type ShredSignatureVerifier struct { mu sync.Mutex cur map[shredSigCacheKey]struct{} prev map[shredSigCacheKey]struct{} @@ -29,6 +35,10 @@ type shredSigCache struct { verifies atomic.Uint64 } +// Keep the internal receiver/test name as an alias; there is one +// implementation and one cache contract. +type shredSigCache = ShredSignatureVerifier + type shredSigCacheKey struct { leader solana.PublicKey root solana.Hash @@ -42,10 +52,17 @@ const shredSigCacheGenCap = 4096 // verifyShred authenticates a shred exactly like Shred.VerifySignature, with // the per-root ed25519 result cached. -func (c *shredSigCache) verifyShred(s *Shred, leader solana.PublicKey) error { +func (c *ShredSignatureVerifier) verifyShred(s *Shred, leader solana.PublicKey) error { + _, err := c.verifyShredRoot(s, leader) + return err +} + +// verifyShredRoot also returns the root authenticated for these exact bytes. +// Callers must keep the shred immutable through assembler admission. +func (c *ShredSignatureVerifier) verifyShredRoot(s *Shred, leader solana.PublicKey) (solana.Hash, error) { root, err := s.MerkleRoot() if err != nil { - return err + return solana.Hash{}, err } key := shredSigCacheKey{leader: leader, root: root, sig: s.Signature} @@ -53,28 +70,34 @@ func (c *shredSigCache) verifyShred(s *Shred, leader solana.PublicKey) error { if _, ok := c.cur[key]; ok { c.mu.Unlock() c.hits.Add(1) - return nil + return root, nil } if _, ok := c.prev[key]; ok { // Promote: a set straddling a rotation keeps its entry hot. c.addLocked(key) c.mu.Unlock() c.hits.Add(1) - return nil + return root, nil } c.mu.Unlock() c.verifies.Add(1) if !narya.VerifyStrict(leader[:], root[:], s.Signature[:]) { - return fmt.Errorf("%w: slot %d shred %d", ErrInvalidSignature, s.Slot, s.Index) + return solana.Hash{}, fmt.Errorf("%w: slot %d shred %d", ErrInvalidSignature, s.Slot, s.Index) } c.mu.Lock() c.addLocked(key) c.mu.Unlock() - return nil + return root, nil } -func (c *shredSigCache) addLocked(key shredSigCacheKey) { +// Verify authenticates one shred and retains successful root/signature tuples +// for sibling shreds in the same FEC set. +func (c *ShredSignatureVerifier) Verify(s *Shred, leader solana.PublicKey) error { + return c.verifyShred(s, leader) +} + +func (c *ShredSignatureVerifier) addLocked(key shredSigCacheKey) { if c.cur == nil { c.cur = make(map[shredSigCacheKey]struct{}, shredSigCacheGenCap) } @@ -85,6 +108,11 @@ func (c *shredSigCache) addLocked(key shredSigCacheKey) { c.cur[key] = struct{}{} } -func (c *shredSigCache) stats() (hits, verifies uint64) { +func (c *ShredSignatureVerifier) stats() (hits, verifies uint64) { return c.hits.Load(), c.verifies.Load() } + +// Stats reports cache hits and actual Ed25519 verifications. +func (c *ShredSignatureVerifier) Stats() (hits, verifies uint64) { + return c.stats() +} diff --git a/pkg/turbine/streaming_message_identity_test.go b/pkg/turbine/streaming_message_identity_test.go new file mode 100644 index 000000000..4fce6debd --- /dev/null +++ b/pkg/turbine/streaming_message_identity_test.go @@ -0,0 +1,104 @@ +package turbine + +import ( + "context" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestMessageIdentitiesPreparedBeforeFinalShred(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + batches := prefetchTestShreds(t, 100, + prefetchTestPayload(t, verifierSignedTransactions(t, 3)), + prefetchTestPayload(t, verifierSignedTransactions(t, 4))) + require.Nil(t, feedPrefetchShreds(t, a, batches[0])) + cached := waitPrefetchedBatch(t, a, 100, 0) + _, err := cached.verification.wait() + require.NoError(t, err) + require.Len(t, cached.verification.identities, 3) + for i := range cached.entries[0].Txns { + tx := &cached.entries[0].Txns[i] + got, ok := cached.verification.identities[i].ForTransaction(tx) + require.True(t, ok) + want, err := txstatus.IdentityForTransaction(tx) + require.NoError(t, err) + require.Equal(t, want, got) + } + blk := feedPrefetchShreds(t, a, batches[1]) + require.NotNil(t, blk) + prepared, err := blk.PrepareTransactionMessageIdentities() + require.NoError(t, err) + for i, tx := range blk.Transactions { + want, err := txstatus.IdentityForTransaction(tx) + require.NoError(t, err) + require.Equal(t, want, prepared.Identity(i)) + } +} + +func TestVerifiedIdentityCacheRejectsMismatchedAndPartialCoverage(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + txs := verifierSignedTransactions(t, 3) + request, err := v.submitTransactions(context.Background(), txs) + require.NoError(t, err) + _, err = request.wait() + require.NoError(t, err) + blk := &block.Block{Transactions: txs} + require.NoError(t, blk.CacheVerifiedTransactionMessageIdentities(request.identities)) + prepared, err := blk.PrepareTransactionMessageIdentities() + require.NoError(t, err) + require.Error(t, blk.CacheVerifiedTransactionMessageIdentities(request.identities[:2])) + copyTx := *txs[0] + for _, changed := range [][]*solana.Transaction{{txs[1], txs[0], txs[2]}, {©Tx, txs[1], txs[2]}} { + other := &block.Block{Transactions: changed} + require.Error(t, other.CacheVerifiedTransactionMessageIdentities(request.identities)) + } + again, err := blk.PrepareTransactionMessageIdentities() + require.NoError(t, err) + require.Same(t, prepared, again, "failed cache adoption must not replace a valid cache") + txs[0].Message.RecentBlockhash[0] ^= 1 + require.Error(t, blk.CacheVerifiedTransactionMessageIdentities(request.identities)) +} + +func TestMessageIdentityFallbackPreservesRetainedTransactionOrder(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + a := NewSlotAssembler() + p := newEntryPrefetchPool(context.Background(), a, v) + defer p.closeAndWait() + batches := prefetchTestShreds(t, 100, + prefetchTestPayload(t, verifierSignedTransactions(t, 3)), + prefetchTestPayload(t, verifierSignedTransactions(t, 4)), + prefetchTestPayload(t, verifierSignedTransactions(t, 5))) + for i := 0; i < 2; i++ { + require.Nil(t, feedPrefetchShreds(t, a, batches[i])) + cached := waitPrefetchedBatch(t, a, 100, batches[i][0].Index) + _, err := cached.verification.wait() + require.NoError(t, err) + if i == 1 { + // Force a canceled, joined result in the middle of the retained + // sequence. Completion must reverify and scatter its identities. + done := make(chan struct{}) + close(done) + cached.verification = &transactionVerification{done: done, err: context.Canceled, index: -1, cancel: func() {}} + } + } + blk := feedPrefetchShreds(t, a, batches[2]) + require.NotNil(t, blk) + require.Len(t, blk.Transactions, 12) + prepared, err := blk.PrepareTransactionMessageIdentities() + require.NoError(t, err) + for i, tx := range blk.Transactions { + want, err := txstatus.IdentityForTransaction(tx) + require.NoError(t, err) + require.Equal(t, want, prepared.Identity(i)) + } +} diff --git a/pkg/turbine/transaction_job_groups_test.go b/pkg/turbine/transaction_job_groups_test.go new file mode 100644 index 000000000..ab2edb554 --- /dev/null +++ b/pkg/turbine/transaction_job_groups_test.go @@ -0,0 +1,118 @@ +package turbine + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestWideVerificationJobsPreserveSignatureFailureIndex(t *testing.T) { + txs := verifierSignedTransactions(t, 320) + txs[33].Signatures[0][0] ^= 1 + txs[98].Signatures[0][0] ^= 1 + for _, groups := range []int{1, 4, 8} { + v := newTransactionVerifierWithJobGroups(2, 32, 8, groups, nil) + r, err := v.submitTransactions(context.Background(), txs) + require.NoError(t, err) + index, err := r.wait() + require.Error(t, err) + require.Equal(t, 33, index) + v.closeAndWait() + } +} + +func TestWideVerificationJobsYieldToReadySmallRequest(t *testing.T) { + for _, groups := range []int{4, 8} { + t.Run(fmt.Sprint(groups), func(t *testing.T) { + large, small := verifierTestBlock(800), verifierTestBlock(4) + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + var calls, beforeSmall atomic.Int32 + v := newTransactionVerifierWithJobGroups(1, 16, 8, groups, func(tx *solana.Transaction) error { + for _, s := range small.Transactions { + if tx == s { + beforeSmall.Store(calls.Load()) + return nil + } + } + calls.Add(1) + if tx == large.Transactions[0] { + close(started) + <-release + } + return nil + }) + defer v.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + big, err := v.submitTransactions(context.Background(), large.Transactions) + require.NoError(t, err) + waitSignal(t, started, "large job") + little, err := v.submitTransactions(context.Background(), small.Transactions) + require.NoError(t, err) + require.Eventually(t, func() bool { return len(v.jobs) == 1 }, 3*time.Second, time.Millisecond) + releaseOnce.Do(func() { close(release) }) + _, err = little.wait() + require.NoError(t, err) + require.Equal(t, int32(groups*8), beforeSmall.Load(), "one large job, not an entire catch-up request, precedes small work") + _, err = big.wait() + require.NoError(t, err) + require.Equal(t, int32(800), calls.Load()) + }) + } +} + +func TestWideVerificationJobsCancelBetweenVectorsAndJoin(t *testing.T) { + for _, groups := range []int{4, 8} { + t.Run(fmt.Sprint(groups), func(t *testing.T) { + started, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + var calls atomic.Int32 + v := newTransactionVerifierWithJobGroups(1, 16, 8, groups, func(*solana.Transaction) error { + if calls.Add(1) == 1 { + close(started) + <-release + } + return nil + }) + defer v.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + r, err := v.submitTransactions(ctx, verifierTestBlock(800).Transactions) + require.NoError(t, err) + waitSignal(t, started, "first vector") + cancel() + select { + case <-r.done: + t.Fatal("released transactions still read by a worker") + default: + } + releaseOnce.Do(func() { close(release) }) + _, err = r.wait() + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, int32(8), calls.Load(), "canceled large job finishes its admitted vector only") + }) + } +} + +func TestWideVerificationJobsDoNotWaitForFourTransactionBatch(t *testing.T) { + for _, groups := range []int{4, 8} { + v := newTransactionVerifierWithJobGroups(2, 32, 8, groups, func(*solana.Transaction) error { return nil }) + r, err := v.submitTransactions(context.Background(), verifierTestBlock(4).Transactions) + require.NoError(t, err) + select { + case <-r.done: + case <-time.After(3 * time.Second): + t.Fatal("partial work waited for another submission") + } + _, err = r.wait() + require.NoError(t, err) + v.closeAndWait() + } +} diff --git a/pkg/turbine/transaction_verifier.go b/pkg/turbine/transaction_verifier.go index e64e36696..050e126b3 100644 --- a/pkg/turbine/transaction_verifier.go +++ b/pkg/turbine/transaction_verifier.go @@ -3,8 +3,8 @@ package turbine import ( "context" "fmt" - "runtime" "sync" + "time" "github.com/Overclock-Validator/mithril/pkg/block" "github.com/Overclock-Validator/mithril/pkg/sigverify" @@ -12,105 +12,340 @@ import ( "github.com/gagliardetto/solana-go" ) -var errNilTransaction = fmt.Errorf("nil transaction") +var ( + errNilTransaction = fmt.Errorf("nil transaction") + errTransactionVerifierClosed = fmt.Errorf("transaction verifier closed") +) + +// Four vector groups amortize dispatch for large ready requests while bounding +// the work that can precede another component on a worker. +const defaultTransactionJobGroups = 4 +// A job is formed before admission, from transactions which are already +// available. Workers never wait for more transactions to fill a vector group. type transactionVerifyJob struct { - tx *solana.Transaction - err *error - done *sync.WaitGroup + trace bool + offeredAt, workerStart, workerEnd int64 + ctx context.Context + txs []*solana.Transaction + identities []txverify.VerifiedMessageIdentity + errs []error + start int + done chan<- *transactionVerifyJob } -type transactionVerifier struct { - jobs chan transactionVerifyJob - verify func(*solana.Transaction) error - workers int - // wave is how many transactions verifyBlockContext admits at once. It is - // workers * sigverify.BatchTarget so each worker can actually accumulate a - // full vector group rather than being handed one transaction at a time. - wave int - close sync.Once +// transactionVerification owns an asynchronous request until done closes. +// Transactions submitted to it must remain immutable until wait returns. +type transactionVerification struct { + trace *entryVerificationTrace + done chan struct{} + cancel context.CancelFunc + index int + err error + finishedAt time.Time + identities []txverify.VerifiedMessageIdentity +} - worker sync.WaitGroup +func (r *transactionVerification) wait() (int, error) { + return r.waitContext(context.Background()) } -func newTransactionVerifier(workers, queueDepth int, verify func(*solana.Transaction) error) *transactionVerifier { - if workers < 1 { - workers = 1 +// waitContext cancels further admission when ctx is canceled, but joins every +// admitted job before returning. The caller may then safely release or mutate +// the transaction objects, including their backing message byte slices. +func (r *transactionVerification) waitContext(ctx context.Context) (int, error) { + if ctx == nil { + ctx = context.Background() + } + select { + case <-r.done: + case <-ctx.Done(): + r.cancel() + <-r.done + return -1, ctx.Err() } - wave := workers * sigverify.BatchTarget - if queueDepth < 1 { - queueDepth = 1 + if err := ctx.Err(); err != nil { + return -1, err } + return r.index, r.err +} + +type transactionVerifier struct { + jobs chan *transactionVerifyJob + verify func(*solana.Transaction) error + workers int + batchTarget int + jobGroups int + // Each accepted request owns at most workers outstanding jobs. The + // admission semaphore also bounds asynchronous request goroutines; callers + // apply backpressure before handing off another decoded component. + requests chan struct{} + // Protected by mu. Reserve one request permit for completion/recovery. + prefetchRequests int + completionWaiters int + admissionChanged chan struct{} + request sync.WaitGroup + mu sync.Mutex + closed bool + stopped chan struct{} + close sync.Once + worker sync.WaitGroup +} + +func newTransactionVerifier(workers, queueDepth int, verify func(*solana.Transaction) error) *transactionVerifier { + return newTransactionVerifierWithBatchTarget(workers, queueDepth, sigverify.BatchTarget, verify) +} + +// queueDepth is a transaction budget, rounded up to whole jobs. batchTarget +// counts signature lanes: multi-signature transactions stay indivisible and +// may exceed the target. Four/eight targets can be compared without changing +// the admission or cancellation policy. +func newTransactionVerifierWithBatchTarget(workers, queueDepth, batchTarget int, verify func(*solana.Transaction) error) *transactionVerifier { + return newTransactionVerifierWithJobGroups(workers, queueDepth, batchTarget, defaultTransactionJobGroups, verify) +} + +// Job groups amortize dispatch over already available vector groups. They do +// not change vector width or wait for future transactions to arrive. +func newTransactionVerifierWithJobGroups(workers, queueDepth, batchTarget, jobGroups int, verify func(*solana.Transaction) error) *transactionVerifier { + workers = max(1, workers) + batchTarget = max(1, min(batchTarget, sigverify.BatchTarget)) + jobGroups = max(1, min(jobGroups, 8)) + jobCapacity := batchTarget * jobGroups + queueGroups := max(1, (queueDepth+jobCapacity-1)/jobCapacity) v := &transactionVerifier{ - jobs: make(chan transactionVerifyJob, queueDepth), - verify: verify, - workers: workers, - wave: wave, + jobs: make(chan *transactionVerifyJob, queueGroups), + verify: verify, + workers: workers, + batchTarget: batchTarget, + jobGroups: jobGroups, + requests: make(chan struct{}, 2*workers), + stopped: make(chan struct{}), } v.worker.Add(workers) for i := 0; i < workers; i++ { go func() { defer v.worker.Done() - // Worker-local scratch reused across groups. - var ( - group []transactionVerifyJob - scr verifyScratch - ) + var batch txverify.BatchVerifier for job := range v.jobs { - group = sigverify.Drain(group, job, v.jobs, - sigverify.FairShare(len(v.jobs), v.workers, sigverify.BatchTarget)) - v.verifyGroup(group, &scr) - // Do not keep finished jobs reachable through the scratch. - clear(group) + v.verifyGroup(job, &batch) } }() } return v } -// verifyScratch is one worker's reusable buffers. -type verifyScratch struct { - txs []*solana.Transaction - errs []error - batch txverify.BatchVerifier -} - -// verifyGroup verifies a drained group and releases every job in it. -// -// Releasing happens in a defer covering the whole group, so no caller can be -// left waiting on a job that was drained into a batch which then failed — -// a stranded job would hang verifyBlockContext's done.Wait() forever. -func (v *transactionVerifier) verifyGroup(group []transactionVerifyJob, scr *verifyScratch) { +// verifyGroup releases its job even if signature verification panics. The +// request's bounded completion channel always has room for every pending job. +func (v *transactionVerifier) verifyGroup(job *transactionVerifyJob, batch *txverify.BatchVerifier) { + if job.trace { + job.workerStart = entryTraceNow() + } defer func() { - for _, job := range group { - job.done.Done() + if job.trace { + job.workerEnd = entryTraceNow() } + job.done <- job }() - - // An injected verifier is a per-transaction function and stays that way; - // only the default path can batch. This seam is used by tests. - if v.verify != nil { - for _, job := range group { - *job.err = verifyTransactionSafely(v.verify, job.tx) + for start := 0; start < len(job.txs); { + // An admitted job always finishes its first vector group, preserving + // ownership/join semantics. Cancellation can skip additional groups. + if err := job.ctx.Err(); start > 0 && err != nil { + for i := start; i < len(job.errs); i++ { + job.errs[i] = err + } + return } - return + end := transactionVerifyGroupEnd(job.txs, start, v.batchTarget) + if v.verify != nil { + for i := start; i < end; i++ { + tx := job.txs[i] + if tx == nil { + job.errs[i] = errNilTransaction + } else { + job.errs[i] = verifyTransactionSafely(v.verify, tx) + } + } + } else { + if job.identities != nil { + verifyBatchWithIdentitiesSafely(batch, job.txs[start:end], job.errs[start:end], job.identities[start:end]) + } else { + verifyBatchSafely(batch, job.txs[start:end], job.errs[start:end]) + } + } + start = end } +} + +// submitTransactions admits one immutable decoded component or complete block. +// Admission is bounded and may block: call it from a decode/completion worker, +// never the UDP reader or while holding the assembler mutex. Cancellation of +// ctx stops further groups but still joins every admitted group. +func (v *transactionVerifier) submitTransactions(ctx context.Context, txs []*solana.Transaction) (*transactionVerification, error) { + return v.submitRequest(ctx, txs, false) +} + +// submitPrefetchTransactions applies backpressure before allocating a request: +// prefetch may use at most 2*workers-1 of the existing 2*workers permits. +func (v *transactionVerifier) submitPrefetchTransactions(ctx context.Context, txs []*solana.Transaction) (*transactionVerification, error) { + return v.submitRequest(ctx, txs, true) +} + +func (v *transactionVerifier) submitRequest(ctx context.Context, txs []*solana.Transaction, prefetch bool) (*transactionVerification, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + var trace *entryVerificationTrace + if entryTraceContext(ctx) { + trace = &entryVerificationTrace{Submit: entryTraceNow(), Transactions: len(txs)} + } + if err := v.acquireRequest(ctx, prefetch); err != nil { + return nil, err + } + ctx, cancel := context.WithCancel(ctx) + if trace != nil { + trace.Admitted = entryTraceNow() + } + r := &transactionVerification{done: make(chan struct{}), cancel: cancel, index: -1, trace: trace} + if v.verify == nil { + r.identities = make([]txverify.VerifiedMessageIdentity, len(txs)) + } + go func() { + defer v.request.Done() + defer v.releaseRequest(prefetch) + defer cancel() + r.index, r.err = v.verifyTransactionsWithTiming(ctx, txs, r.identities, trace) + if trace != nil { + trace.Finished = entryTraceNow() + } + r.finishedAt = time.Now() + close(r.done) + }() + return r, nil +} + +// verifyTransactions keeps a rolling window instead of waiting for an entire +// worker wave. A slow job cannot idle workers whose earlier jobs finished. +// One caller can queue at most workers jobs, so a large catch-up block cannot +// put all its transactions ahead of a newly available component. +func (v *transactionVerifier) verifyTransactions(ctx context.Context, txs []*solana.Transaction) (int, error) { + return v.verifyTransactionsWithIdentities(ctx, txs, nil) +} - scr.txs = scr.txs[:0] - for _, job := range group { - scr.txs = append(scr.txs, job.tx) +func (v *transactionVerifier) verifyTransactionsWithIdentities(ctx context.Context, txs []*solana.Transaction, identities []txverify.VerifiedMessageIdentity) (int, error) { + return v.verifyTransactionsWithTiming(ctx, txs, identities, nil) +} + +func (v *transactionVerifier) verifyTransactionsWithTiming(ctx context.Context, txs []*solana.Transaction, identities []txverify.VerifiedMessageIdentity, trace *entryVerificationTrace) (int, error) { + if len(txs) == 0 { + return -1, ctx.Err() + } + window := min(v.workers, len(txs)) + jobGroups := v.jobGroups + // Keep short components responsive and enough independent jobs to supply + // every worker. This is a ready-work threshold, never a batching timer. + if len(txs) < 2*v.workers*v.batchTarget*jobGroups { + jobGroups = 1 } - if cap(scr.errs) < len(scr.txs) { - scr.errs = make([]error, len(scr.txs)) + jobCapacity := v.batchTarget * jobGroups + completed := make(chan *transactionVerifyJob, window) + groups := make([]transactionVerifyJob, window) + errs := make([]error, window*jobCapacity) + free := make([]*transactionVerifyJob, window) + for i := range groups { + groups[i].trace = trace != nil + groups[i].ctx = ctx + groups[i].errs = errs[i*jobCapacity : (i+1)*jobCapacity] + groups[i].done = completed + free[i] = &groups[i] } - scr.errs = scr.errs[:len(scr.txs)] - verifyBatchSafely(&scr.batch, scr.txs, scr.errs) + nextIndex, active := 0, 0 + failureIndex := -1 + var failure error + var pending *transactionVerifyJob + ctxDone := ctx.Done() + stopped := false + for active > 0 || (!stopped && nextIndex < len(txs)) { + if !stopped && ctx.Err() != nil { + stopped = true + ctxDone = nil + } + if stopped && active == 0 { + break + } + if !stopped && pending == nil && nextIndex < len(txs) && len(free) > 0 { + pending = free[len(free)-1] + free = free[:len(free)-1] + end := nextIndex + for group := 0; group < jobGroups && end < len(txs); group++ { + end = transactionVerifyGroupEnd(txs, end, v.batchTarget) + } + if trace != nil { + pending.offeredAt = entryTraceNow() + } + pending.start = nextIndex + pending.txs = txs[nextIndex:end] + if identities != nil { + pending.identities = identities[nextIndex:end] + } + pending.errs = pending.errs[:end-nextIndex] + clear(pending.errs) + } + var admission chan *transactionVerifyJob + if !stopped && pending != nil { + admission = v.jobs + } + select { + case admission <- pending: + nextIndex += len(pending.txs) + active++ + pending = nil + case job := <-completed: + if trace != nil { + trace.observe(job) + } + active-- + for i, err := range job.errs { + if err != nil && (failureIndex < 0 || job.start+i < failureIndex) { + failureIndex, failure = job.start+i, err + stopped = true + } + } + job.txs = nil + job.identities = nil + clear(job.errs) + free = append(free, job) + case <-ctxDone: + stopped = true + ctxDone = nil + } + } + if err := ctx.Err(); err != nil { + return -1, err + } + return failureIndex, failure +} - for i, job := range group { - *job.err = scr.errs[i] +func transactionVerifyGroupEnd(txs []*solana.Transaction, start, target int) int { + end, signatures := start, 0 + for end < len(txs) && end-start < target { + count := 1 + if txs[end] != nil { + count = max(1, len(txs[end].Signatures)) + } + if end > start && signatures+count > target { + break + } + signatures += count + end++ + if signatures >= target { + break + } } - clear(scr.txs) + return end } // verifyBatchSafely mirrors verifyTransactionSafely: a panic in the verifier @@ -129,11 +364,28 @@ func verifyBatchSafely(batch *txverify.BatchVerifier, txs []*solana.Transaction, batch.Verify(txs, errs) } +func verifyBatchWithIdentitiesSafely(batch *txverify.BatchVerifier, txs []*solana.Transaction, errs []error, identities []txverify.VerifiedMessageIdentity) { + defer func() { + if recovered := recover(); recovered != nil { + clear(identities) + for i := range errs { + errs[i] = fmt.Errorf("signature verifier panic: %v", recovered) + } + } + }() + batch.VerifyWithMessageIdentities(txs, errs, identities) +} + func (v *transactionVerifier) closeAndWait() { if v == nil { return } v.close.Do(func() { + v.mu.Lock() + v.closed = true + close(v.stopped) + v.mu.Unlock() + v.request.Wait() close(v.jobs) v.worker.Wait() }) @@ -162,47 +414,18 @@ func (v *transactionVerifier) verifyBlockContext(ctx context.Context, blk *block if blk == nil || len(blk.Transactions) == 0 { return nil } - // Admit one worker-wave at a time. A monster block still occupies every - // verifier lane, but cannot park tens of thousands of jobs ahead of a newly - // completed small block in the shared bounded queue. - for chunkStart := 0; chunkStart < len(blk.Transactions); chunkStart += v.wave { - if err := ctx.Err(); err != nil { - return err - } - chunkEnd := min(chunkStart+v.wave, len(blk.Transactions)) - errs := make([]error, chunkEnd-chunkStart) - var done sync.WaitGroup - for txIdx := chunkStart; txIdx < chunkEnd; txIdx++ { - if err := ctx.Err(); err != nil { - done.Wait() - return err - } - tx := blk.Transactions[txIdx] - errIdx := txIdx - chunkStart - if tx == nil { - errs[errIdx] = errNilTransaction - continue - } - done.Add(1) - select { - case v.jobs <- transactionVerifyJob{tx: tx, err: &errs[errIdx], done: &done}: - case <-ctx.Done(): - done.Done() - done.Wait() - return ctx.Err() - } - } - done.Wait() - if err := ctx.Err(); err != nil { - return err - } - for errIdx, err := range errs { - if err != nil { - return formatTransactionVerificationError(blk, chunkStart+errIdx, err) - } - } + request, err := v.submitTransactions(ctx, blk.Transactions) + if err != nil { + return err + } + index, err := request.wait() + if err != nil && index >= 0 { + return formatTransactionVerificationError(blk, index, err) } - return nil + if err == nil && request.identities != nil { + return blk.CacheVerifiedTransactionMessageIdentities(request.identities) + } + return err } func formatTransactionVerificationError(blk *block.Block, txIdx int, err error) error { @@ -226,12 +449,17 @@ var ( defaultTransactionVerifier *transactionVerifier ) -func validateBlockTransactionsContext(ctx context.Context, blk *block.Block) error { +func getDefaultTransactionVerifier() *transactionVerifier { defaultTransactionVerifierOnce.Do(func() { - workers := max(1, (runtime.GOMAXPROCS(0)+1)/2) - defaultTransactionVerifier = newTransactionVerifier(workers, 2*workers*sigverify.BatchTarget, nil) + workers := sigverify.TransactionWorkers() + target := sigverify.TransactionBatchTarget() + defaultTransactionVerifier = newTransactionVerifierWithBatchTarget(workers, 2*workers*target, target, nil) }) - return defaultTransactionVerifier.verifyBlockContext(ctx, blk) + return defaultTransactionVerifier +} + +func validateBlockTransactionsContext(ctx context.Context, blk *block.Block) error { + return getDefaultTransactionVerifier().verifyBlockContext(ctx, blk) } func validateBlockTransactions(blk *block.Block) error { diff --git a/pkg/turbine/transaction_verifier_admission.go b/pkg/turbine/transaction_verifier_admission.go new file mode 100644 index 000000000..c0a20ed2c --- /dev/null +++ b/pkg/turbine/transaction_verifier_admission.go @@ -0,0 +1,76 @@ +package turbine + +import "context" + +// acquireRequest keeps the total request/job bounds unchanged while reserving +// one permit for completion or full-block recovery. Waiting completions win the +// next available permit over prefetch; completions are otherwise equal priority. +// This is not replay-head scheduling: unfinished prefetch already admitted for +// the head keeps its existing rolling job window, and future-slot completions +// also use the reservation. No worker is reserved and no admitted job is evicted. +// +// Prefetch can wait while completions remain queued. It is speculative work and +// resumes when the completion backlog drains. Cancellation/close wake waiters +// without admitting a request; accepted requests retain the full join contract. +func (v *transactionVerifier) acquireRequest(ctx context.Context, prefetch bool) error { + v.mu.Lock() + waitingCompletion := false + defer func() { + if waitingCompletion { + v.completionWaiters-- + v.wakeAdmissionLocked() + } + v.mu.Unlock() + }() + for { + if v.closed { + return errTransactionVerifierClosed + } + if err := ctx.Err(); err != nil { + return err + } + if len(v.requests) < cap(v.requests) && + (!prefetch || (v.prefetchRequests < cap(v.requests)-1 && v.completionWaiters == 0)) { + v.requests <- struct{}{} + if prefetch { + v.prefetchRequests++ + } + // Add under the same lock as close, so closeAndWait cannot finish + // while an accepted request has yet to start its goroutine. + v.request.Add(1) + return nil + } + if !prefetch && !waitingCompletion { + v.completionWaiters++ + waitingCompletion = true + } + if v.admissionChanged == nil { + v.admissionChanged = make(chan struct{}) + } + changed := v.admissionChanged + v.mu.Unlock() + select { + case <-changed: + case <-ctx.Done(): + case <-v.stopped: + } + v.mu.Lock() + } +} + +func (v *transactionVerifier) releaseRequest(prefetch bool) { + v.mu.Lock() + <-v.requests + if prefetch { + v.prefetchRequests-- + } + v.wakeAdmissionLocked() + v.mu.Unlock() +} + +func (v *transactionVerifier) wakeAdmissionLocked() { + if v.admissionChanged != nil { + close(v.admissionChanged) + v.admissionChanged = nil + } +} diff --git a/pkg/turbine/transaction_verifier_admission_benchmark_test.go b/pkg/turbine/transaction_verifier_admission_benchmark_test.go new file mode 100644 index 000000000..9971224f9 --- /dev/null +++ b/pkg/turbine/transaction_verifier_admission_benchmark_test.go @@ -0,0 +1,118 @@ +package turbine + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/gagliardetto/solana-go" +) + +// Compare the request-class policy with the same workers, rolling job window, +// Narya backend and total work. Shared submits prefetch as ordinary completion +// requests to reproduce the old four-permit occupancy; reserved labels it as +// prefetch. This is a saturation microbenchmark, not observed live p99 or true +// replay-head scheduling. All signatures are verified and every request joined. +func BenchmarkVerifierCompletionReservation(b *testing.B) { + flowConfigureBackend(b) + txs := make([]*solana.Transaction, 4096) + for i := range txs { + txs[i] = flowGeneratedTransaction(b, 228, uint64(i)) + } + for _, size := range []int{256, 4096} { + for _, reserved := range []bool{false, true} { + b.Run(fmt.Sprintf("prefetch_%d/reserved_%t", size, reserved), func(b *testing.B) { + v := newTransactionVerifierWithBatchTarget(2, 32, 8, nil) + defer v.closeAndWait() + submit := v.submitTransactions + if reserved { + submit = v.submitPrefetchTransactions + } + var admission, finish, total []time.Duration + occupied := 0 + ctx := context.WithValue(context.Background(), entryTraceContextKey{}, true) + warm, err := v.submitTransactions(context.Background(), txs[:256]) + if err != nil { + b.Fatal(err) + } + if _, err = warm.wait(); err != nil { + b.Fatal(err) + } + b.ResetTimer() + for range b.N { + start := time.Now() + prior := make([]*transactionVerification, 0, 4) + for range 3 { + r, err := submit(context.Background(), txs[:size]) + if err != nil { + b.Fatal(err) + } + prior = append(prior, r) + } + type result struct { + r *transactionVerification + err error + } + fourth := make(chan result, 1) + attempting := make(chan struct{}) + go func() { + close(attempting) + r, err := submit(context.Background(), txs[:size]) + fourth <- result{r, err} + }() + <-attempting + if !reserved { + last := <-fourth + if last.err != nil { + b.Fatal(last.err) + } + prior = append(prior, last.r) + } + expectedActive := cap(v.requests) + if reserved { + expectedActive-- + } + if len(v.requests) == expectedActive { + occupied++ + } + r, err := v.submitTransactions(ctx, txs[:32]) + if err != nil { + b.Fatal(err) + } + if _, err = r.wait(); err != nil { + b.Fatal(err) + } + admission = append(admission, time.Duration(r.trace.Admitted-r.trace.Submit)) + finish = append(finish, time.Duration(r.trace.Finished-r.trace.Submit)) + if reserved { + last := <-fourth + if last.err != nil { + b.Fatal(last.err) + } + prior = append(prior, last.r) + } + for _, r := range prior { + if _, err = r.wait(); err != nil { + b.Fatal(err) + } + } + total = append(total, time.Since(start)) + } + b.StopTimer() + b.ReportMetric(float64(occupied)/float64(b.N), "occupied_fraction") + for _, metric := range []struct { + name string + values []time.Duration + }{ + {"completion_admit", admission}, {"completion_done", finish}, {"all_work", total}, + } { + flowReportPercentiles(b, metric.values, metric.name) + index := max(0, (len(metric.values)*99+99)/100-1) + b.ReportMetric(float64(metric.values[index])/float64(time.Millisecond), metric.name+"_p99-ms") + b.ReportMetric(float64(metric.values[len(metric.values)-1])/float64(time.Millisecond), metric.name+"_max-ms") + } + }) + } + } +} diff --git a/pkg/turbine/transaction_verifier_admission_test.go b/pkg/turbine/transaction_verifier_admission_test.go new file mode 100644 index 000000000..eef2c9604 --- /dev/null +++ b/pkg/turbine/transaction_verifier_admission_test.go @@ -0,0 +1,172 @@ +package turbine + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestPrefetchLeavesCompletionPermitAndCancellationJoins(t *testing.T) { + release := make(chan struct{}) + v := newTransactionVerifier(2, 32, func(*solana.Transaction) error { <-release; return nil }) + defer v.closeAndWait() + defer close(release) + for range 3 { + _, err := v.submitPrefetchTransactions(context.Background(), verifierTestBlock(64).Transactions) + require.NoError(t, err) + } + require.Equal(t, 3, len(v.requests)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + blocked := make(chan error, 1) + go func() { _, err := v.submitPrefetchTransactions(ctx, verifierTestBlock(1).Transactions); blocked <- err }() + accepted := make(chan *transactionVerification, 1) + go func() { + r, err := v.submitTransactions(context.Background(), verifierTestBlock(1).Transactions) + if err != nil { + t.Error(err) + } + accepted <- r + }() + select { + case r := <-accepted: + require.NotNil(t, r) + case <-time.After(3 * time.Second): + t.Fatal("prefetch occupied the reserved completion permit") + } + require.Equal(t, cap(v.requests), len(v.requests), "total bound must not increase") + cancel() + select { + case err := <-blocked: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(3 * time.Second): + t.Fatal("canceled prefetch admission did not return") + } + // The admitted jobs are still reading transactions until release closes. + require.Equal(t, 4, len(v.requests)) +} + +// Exercise permit arbitration without depending on cryptographic job duration. +// Holding permits models admitted requests; each release also joins its request. +func TestCompletionWinsAdmissionAndPrefetchResumes(t *testing.T) { + v := newTransactionVerifier(1, 8, nil) + ctx, cancel := context.WithCancel(context.Background()) + var cleanup sync.WaitGroup + defer v.closeAndWait() + defer cleanup.Wait() + defer cancel() + release := func(prefetch bool) { v.releaseRequest(prefetch); v.request.Done() } + for range 2 { + require.NoError(t, v.acquireRequest(ctx, false)) + } + firstReleased := false + defer func() { + if !firstReleased { + release(false) + } + release(false) + }() + completionAdmitted := make(chan struct{}) + completionRelease := make(chan struct{}) + var once sync.Once + defer once.Do(func() { close(completionRelease) }) + cleanup.Go(func() { + if err := v.acquireRequest(ctx, false); err != nil { + return + } + close(completionAdmitted) + select { + case <-completionRelease: + case <-ctx.Done(): + } + release(false) + }) + require.Eventually(t, func() bool { v.mu.Lock(); defer v.mu.Unlock(); return v.completionWaiters == 1 }, 3*time.Second, time.Millisecond) + prefetchAdmitted := make(chan struct{}) + cleanup.Go(func() { + if err := v.acquireRequest(ctx, true); err != nil { + return + } + close(prefetchAdmitted) + release(true) + }) + release(false) + firstReleased = true + waitSignal(t, completionAdmitted, "priority completion admission") + select { + case <-prefetchAdmitted: + t.Fatal("prefetch bypassed waiting completion") + default: + } + once.Do(func() { close(completionRelease) }) + waitSignal(t, prefetchAdmitted, "prefetch resumed after completion") +} + +func TestCanceledCompletionDoesNotBlockPrefetch(t *testing.T) { + v := newTransactionVerifier(1, 8, nil) + defer v.closeAndWait() + for range 2 { + require.NoError(t, v.acquireRequest(context.Background(), false)) + } + defer func() { + for range 2 { + v.releaseRequest(false) + v.request.Done() + } + }() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- v.acquireRequest(ctx, false) }() + require.Eventually(t, func() bool { v.mu.Lock(); defer v.mu.Unlock(); return v.completionWaiters == 1 }, 3*time.Second, time.Millisecond) + cancel() + select { + case err := <-done: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(3 * time.Second): + t.Fatal("completion cancellation stranded admission") + } + v.mu.Lock() + require.Zero(t, v.completionWaiters) + v.mu.Unlock() +} + +func TestCloseWakesBothAdmissionClassesAndJoinsAcceptedWork(t *testing.T) { + release := make(chan struct{}) + v := newTransactionVerifier(1, 8, func(*solana.Transaction) error { <-release; return nil }) + var once sync.Once + defer v.closeAndWait() + defer once.Do(func() { close(release) }) + _, err := v.submitPrefetchTransactions(context.Background(), verifierTestBlock(1).Transactions) + require.NoError(t, err) + _, err = v.submitTransactions(context.Background(), verifierTestBlock(1).Transactions) + require.NoError(t, err) + results := make(chan error, 2) + for _, prefetch := range []bool{true, false} { + go func() { + _, err := v.submitRequest(context.Background(), verifierTestBlock(1).Transactions, prefetch) + results <- err + }() + } + closed := make(chan struct{}) + go func() { v.closeAndWait(); close(closed) }() + for range 2 { + select { + case err := <-results: + require.ErrorIs(t, err, errTransactionVerifierClosed) + case <-time.After(3 * time.Second): + t.Fatal("close stranded admission") + } + } + select { + case <-closed: + t.Fatal("close returned while jobs still own transactions") + default: + } + once.Do(func() { close(release) }) + waitSignal(t, closed, "close joined accepted work") +} diff --git a/pkg/turbine/transaction_verifier_flow_benchmark_test.go b/pkg/turbine/transaction_verifier_flow_benchmark_test.go new file mode 100644 index 000000000..0c8ce131b --- /dev/null +++ b/pkg/turbine/transaction_verifier_flow_benchmark_test.go @@ -0,0 +1,479 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + "syscall" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/sigverify" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" +) + +// BenchmarkTransactionVerificationFlow measures the real verifier pool under +// simulated transaction availability, not actual network reception or replay. +// Decode and fixture generation are outside the timer. "tip" spreads complete +// components over 200 ms; this is an explicit workload model, not a claim about +// the cluster's observed component sizes or arrival distribution. No component +// waits for another component to fill a verification batch. +// +// Optional environment: +// +// MITHRIL_SIGVERIFY_FLOW_BACKEND=r51 (use -run '^$' in a fresh test process) +// MITHRIL_SIGVERIFY_FLOW_FIXTURES=/path/to/fixtures (block-*.json, base64 txs) +// MITHRIL_SIGVERIFY_FLOW_COUNT=33760 (generated count or captured prefix limit) +// +// Without captured fixtures, use reproducible signed 228-byte and 1232-byte +// legacy memo transactions. They are signature workloads, not replay fixtures. +// Use -benchtime=3x or another fixed count when comparing configurations so the +// deliberate arrival waits do not change the number of observations. +func BenchmarkTransactionVerificationFlow(b *testing.B) { + flowConfigureBackend(b) + for _, fixture := range flowBenchmarkFixtures(b) { + b.Run(fixture.name, func(b *testing.B) { + for _, workers := range []int{2, 4} { + b.Run(fmt.Sprintf("workers_%d", workers), func(b *testing.B) { + for _, target := range []int{4, 8} { + b.Run(fmt.Sprintf("target_%d", target), func(b *testing.B) { + for _, scenario := range flowScenarios(fixture.blk) { + b.Run(scenario.name, func(b *testing.B) { + flowRunBenchmark(b, workers, target, fixture.blk, scenario) + }) + } + }) + } + }) + } + }) + } +} + +type flowFixture struct { + name string + blk *block.Block +} + +type flowScenario struct { + name string + components [][]*solana.Transaction + arrivalSpan time.Duration + overlap bool +} + +func flowScenarios(blk *block.Block) []flowScenario { + // 60 KiB is a benchmark parameter only. Include signed transaction bytes; + // actual serialized entry/component overhead and shred recovery are omitted. + components := flowComponents(blk.Transactions, 60*1024) + scenarios := []flowScenario{ + {name: "catchup", components: [][]*solana.Transaction{blk.Transactions}}, + {name: "tip_200ms_after_complete", components: components, arrivalSpan: 200 * time.Millisecond}, + {name: "tip_200ms_overlap", components: components, arrivalSpan: 200 * time.Millisecond, overlap: true}, + } + // Sparse components expose tail behavior without requiring microsecond + // timer precision to deliver tens of thousands of tiny arrival events. + for _, width := range []int{4, 7, 8} { + txs := blk.Transactions[:min(256, len(blk.Transactions))] + var small [][]*solana.Transaction + for start := 0; start < len(txs); start += width { + small = append(small, txs[start:min(start+width, len(txs))]) + } + scenarios = append(scenarios, flowScenario{ + name: fmt.Sprintf("sparse_%dtx_200ms_overlap", width), components: small, + arrivalSpan: 200 * time.Millisecond, overlap: true, + }) + } + return scenarios +} + +func flowComponents(txs []*solana.Transaction, bytesPerComponent int) [][]*solana.Transaction { + var components [][]*solana.Transaction + start, size := 0, 0 + for i, tx := range txs { + wireSize, err := txverify.TransactionWireSize(tx) + if err != nil { + panic(err) // fixtures are validated before entering this helper + } + if i > start && size+wireSize > bytesPerComponent { + components = append(components, txs[start:i]) + start, size = i, 0 + } + size += wireSize + } + if start < len(txs) { + components = append(components, txs[start:]) + } + return components +} + +type flowObservation struct { + latencies []time.Duration + submits []time.Duration + feedLags []time.Duration + residual time.Duration + err error +} + +func flowArrivalOffset(index, count int, span time.Duration) time.Duration { + if count <= 1 { + return span + } + return time.Duration(int64(span) * int64(index) / int64(count-1)) +} + +func flowObserve(v *transactionVerifier, blk *block.Block, scenario flowScenario) flowObservation { + observation := flowObservation{ + latencies: make([]time.Duration, len(scenario.components)), + submits: make([]time.Duration, len(scenario.components)), + feedLags: make([]time.Duration, len(scenario.components)), + } + started := time.Now() + finalArrival := started.Add(scenario.arrivalSpan) + if !scenario.overlap { + time.Sleep(time.Until(finalArrival)) + submitStarted := time.Now() + future, err := v.submitTransactions(context.Background(), blk.Transactions) + submitDuration := time.Since(submitStarted) + if err != nil { + observation.err = err + return observation + } + _, observation.err = future.wait() + finished := future.finishedAt + for i := range observation.latencies { + available := started.Add(flowArrivalOffset(i, len(scenario.components), scenario.arrivalSpan)) + observation.latencies[i] = finished.Sub(available) + observation.submits[i] = submitDuration + observation.feedLags[i] = submitStarted.Sub(available) + } + observation.residual = max(0, finished.Sub(finalArrival)) + return observation + } + + var waiters sync.WaitGroup + errs := make([]error, len(scenario.components)) + finished := make([]time.Time, len(scenario.components)) + for i, txs := range scenario.components { + available := started.Add(flowArrivalOffset(i, len(scenario.components), scenario.arrivalSpan)) + time.Sleep(time.Until(available)) + submitStarted := time.Now() + observation.feedLags[i] = submitStarted.Sub(available) + future, err := v.submitTransactions(context.Background(), txs) + observation.submits[i] = time.Since(submitStarted) + if err != nil { + errs[i] = err + break + } + waiters.Add(1) + go func() { + defer waiters.Done() + _, errs[i] = future.wait() + finished[i] = future.finishedAt + observation.latencies[i] = finished[i].Sub(available) + }() + } + waiters.Wait() + for _, completed := range finished { + observation.residual = max(observation.residual, completed.Sub(finalArrival)) + } + for _, err := range errs { + if err != nil { + observation.err = err + break + } + } + return observation +} + +func flowRunBenchmark(b *testing.B, workers, target int, blk *block.Block, scenario flowScenario) { + v := newTransactionVerifierWithBatchTarget(workers, 2*workers*8, target, nil) + defer v.closeAndWait() + if err := v.verifyBlock(blk); err != nil { + b.Fatal(err) + } + flowBenchmarkGate(b) + var signatureCount int + for _, component := range scenario.components { + for _, tx := range component { + signatureCount += len(tx.Signatures) + } + } + var latencies, submits, feedLags, residuals []time.Duration + before := sigverify.Stats() + cpuBefore := flowCPUSeconds(b) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + observation := flowObserve(v, blk, scenario) + if observation.err != nil { + b.Fatal(observation.err) + } + latencies = append(latencies, observation.latencies...) + submits = append(submits, observation.submits...) + feedLags = append(feedLags, observation.feedLags...) + residuals = append(residuals, observation.residual) + } + b.StopTimer() + cpuSeconds := flowCPUSeconds(b) - cpuBefore + after := sigverify.Stats() + b.ReportMetric(1000*cpuSeconds/float64(b.N), "cpu-ms/block") + b.ReportMetric(cpuSeconds/b.Elapsed().Seconds(), "avg_cpu_cores") + b.ReportMetric(float64(b.N*signatureCount)/b.Elapsed().Seconds(), "signatures/s") + b.ReportMetric(float64(signatureCount), "signatures/block") + b.ReportMetric(float64(len(scenario.components)), "components/block") + b.ReportMetric(float64(after.Signatures-before.Signatures)/float64(after.Batches-before.Batches), "mean_width") + flowReportPercentiles(b, latencies, "ready") + flowReportPercentiles(b, submits, "submit") + flowReportPercentiles(b, feedLags, "feed_lag") + flowReportPercentiles(b, residuals, "residual") + if after.InternalFaultFallbacks != before.InternalFaultFallbacks { + b.Fatal("signature verifier used an internal fault fallback") + } + if want := uint64(b.N * signatureCount); after.Signatures-before.Signatures != want { + b.Fatalf("verified signature count = %d, want %d", after.Signatures-before.Signatures, want) + } +} + +// The optional rendezvous is outside the timer. An external contention runner +// starts the real execution probe after seeing READY, then creates START. Both +// paths are explicit files in that runner's output directory. This is only for +// coordination; ordinary benchmark invocations do no filesystem polling. +func flowBenchmarkGate(b *testing.B) { + b.Helper() + ready, start := os.Getenv("MITHRIL_SIGVERIFY_FLOW_READY"), os.Getenv("MITHRIL_SIGVERIFY_FLOW_START") + if ready == "" && start == "" { + return + } + if ready == "" || start == "" { + b.Fatal("set both MITHRIL_SIGVERIFY_FLOW_READY and MITHRIL_SIGVERIFY_FLOW_START") + } + // Go's first N=1 calibration and warmup must finish before the external + // execution probe starts. The runner selects a fixed N greater than one. + if b.N == 1 { + return + } + if err := os.WriteFile(ready, []byte(b.Name()+"\n"), 0600); err != nil { + b.Fatal(err) + } + deadline := time.Now().Add(30 * time.Second) + for { + if _, err := os.Stat(start); err == nil { + return + } else if !os.IsNotExist(err) { + b.Fatal(err) + } + if time.Now().After(deadline) { + b.Fatal("contention runner did not release benchmark within 30 seconds") + } + time.Sleep(time.Millisecond) + } +} + +func flowReportPercentiles(b *testing.B, values []time.Duration, prefix string) { + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + for _, percentile := range []int{50, 95} { + index := max(0, (len(values)*percentile+99)/100-1) + b.ReportMetric(float64(values[index])/float64(time.Millisecond), fmt.Sprintf("%s_p%d-ms", prefix, percentile)) + } +} + +func flowCPUSeconds(tb testing.TB) float64 { + tb.Helper() + var usage syscall.Rusage + if err := syscall.Getrusage(syscall.RUSAGE_SELF, &usage); err != nil { + tb.Fatal(err) + } + return float64(usage.Utime.Sec+usage.Stime.Sec) + float64(usage.Utime.Usec+usage.Stime.Usec)/1e6 +} + +var flowBackendOnce sync.Once +var flowBackendError error + +func flowConfigureBackend(tb testing.TB) { + tb.Helper() + flowBackendOnce.Do(func() { + if backend := os.Getenv("MITHRIL_SIGVERIFY_FLOW_BACKEND"); backend != "" { + _, flowBackendError = sigverify.Configure(sigverify.Config{Backend: backend}) + } + }) + if flowBackendError != nil { + tb.Fatal(flowBackendError) + } +} + +func flowBenchmarkFixtures(tb testing.TB) []flowFixture { + tb.Helper() + count := 33760 + limit := false + if value := os.Getenv("MITHRIL_SIGVERIFY_FLOW_COUNT"); value != "" { + var err error + count, err = strconv.Atoi(value) + if err != nil || count < 1 { + tb.Fatal("MITHRIL_SIGVERIFY_FLOW_COUNT must be a positive integer") + } + limit = true + } + if dir := os.Getenv("MITHRIL_SIGVERIFY_FLOW_FIXTURES"); dir != "" { + paths, err := filepath.Glob(filepath.Join(dir, "block-*.json")) + if err != nil || len(paths) == 0 { + tb.Fatalf("captured fixtures: %v, files=%d", err, len(paths)) + } + var fixtures []flowFixture + for _, path := range paths { + data, err := os.ReadFile(path) + if err != nil { + tb.Fatal(err) + } + var captured struct { + Slot uint64 + Transactions []string + } + if err := json.Unmarshal(data, &captured); err != nil { + tb.Fatal(err) + } + blk := &block.Block{Slot: captured.Slot} + for i, encoded := range captured.Transactions { + if limit && i == count { + break + } + wire, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + tb.Fatal(err) + } + tx, err := solana.TransactionFromBytes(wire) + if err != nil { + tb.Fatal(err) + } + if err := txverify.SanitizeTransaction(tx); err != nil { + tb.Fatal(err) + } + blk.Transactions = append(blk.Transactions, tx) + } + if len(blk.Transactions) == 0 { + tb.Fatalf("fixture %s contains no transactions", path) + } + fixtures = append(fixtures, flowFixture{fmt.Sprintf("captured_%d", blk.Slot), blk}) + } + return fixtures + } + var fixtures []flowFixture + for _, wireSize := range []int{228, txverify.MaxLegacyTransactionSize} { + blk := &block.Block{Slot: 1, Transactions: make([]*solana.Transaction, count)} + for i := range blk.Transactions { + blk.Transactions[i] = flowGeneratedTransaction(tb, wireSize, uint64(i)) + } + fixtures = append(fixtures, flowFixture{fmt.Sprintf("generated_%dB", wireSize), blk}) + } + return fixtures +} + +func flowGeneratedTransaction(tb testing.TB, wireSize int, index uint64) *solana.Transaction { + tb.Helper() + // Public, deterministic benchmark material, never a validator identity. + var seed [ed25519.SeedSize]byte + binary.LittleEndian.PutUint64(seed[:], index+1) + private := ed25519.NewKeyFromSeed(seed[:]) + public := solana.PublicKeyFromBytes(private.Public().(ed25519.PublicKey)) + tx := &solana.Transaction{ + Signatures: make([]solana.Signature, 1), + Message: solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1, NumReadonlyUnsignedAccounts: 1}, + AccountKeys: []solana.PublicKey{public, solana.MemoProgramID}, + Instructions: []solana.CompiledInstruction{{ProgramIDIndex: 1, Accounts: []uint16{0}}}, + }, + } + binary.LittleEndian.PutUint64(tx.Message.RecentBlockhash[:], index+1) + baseSize, err := txverify.TransactionWireSize(tx) + if err != nil { + tb.Fatal(err) + } + padding := wireSize - baseSize + for attempts := 0; attempts < 3 && padding >= 0; attempts++ { + tx.Message.Instructions[0].Data = make([]byte, padding) + size, err := txverify.TransactionWireSize(tx) + if err != nil { + tb.Fatal(err) + } + if size != wireSize { + padding += wireSize - size + continue + } + for i := range tx.Message.Instructions[0].Data { + tx.Message.Instructions[0].Data[i] = 'a' + } + message, err := txverify.MessageBytes(tx) + if err != nil { + tb.Fatal(err) + } + copy(tx.Signatures[0][:], ed25519.Sign(private, message)) + if err := txverify.SanitizeTransaction(tx); err != nil { + tb.Fatal(err) + } + return tx + } + tb.Fatalf("cannot construct a %d-byte transaction", wireSize) + return nil +} + +func TestTransactionVerificationFlowFixtureShape(t *testing.T) { + for _, size := range []int{228, 1232} { + first := flowGeneratedTransaction(t, size, 0) + second := flowGeneratedTransaction(t, size, 1) + for _, tx := range []*solana.Transaction{first, second} { + wire, err := tx.MarshalBinary() + if err != nil { + t.Fatal(err) + } + if len(wire) != size || len(tx.Signatures) != 1 { + t.Fatalf("fixture wire=%d signatures=%d, want %d bytes and one signature", len(wire), len(tx.Signatures), size) + } + if err := txverify.VerifyTransaction(tx); err != nil { + t.Fatal(err) + } + } + if first.Signatures[0] == second.Signatures[0] || first.Message.AccountKeys[0] == second.Message.AccountKeys[0] { + t.Fatal("different fixture indices reused a message signature or signer") + } + } +} + +func TestTransactionVerificationFlowComponentBoundaries(t *testing.T) { + txs := make([]*solana.Transaction, 7) + for i := range txs { + txs[i] = flowGeneratedTransaction(t, 228, uint64(i)) + } + components := flowComponents(txs, 500) + if len(components) != 4 { + t.Fatalf("got %d components, want four", len(components)) + } + next := 0 + for i, component := range components { + want := 2 + if i == 3 { + want = 1 + } + if len(component) != want { + t.Fatalf("component %d contains %d transactions, want %d", i, len(component), want) + } + for _, tx := range component { + if tx != txs[next] { + t.Fatal("component split changed transaction order or identity") + } + next++ + } + } + if flowArrivalOffset(0, 4, 200*time.Millisecond) != 0 || flowArrivalOffset(3, 4, 200*time.Millisecond) != 200*time.Millisecond { + t.Fatal("availability schedule does not span the requested interval") + } +} diff --git a/pkg/turbine/transaction_verifier_test.go b/pkg/turbine/transaction_verifier_test.go index 8eb19c63d..dbb73e859 100644 --- a/pkg/turbine/transaction_verifier_test.go +++ b/pkg/turbine/transaction_verifier_test.go @@ -1,9 +1,12 @@ package turbine import ( + "context" + "crypto/ed25519" "errors" "fmt" "strings" + "sync" "sync/atomic" "testing" "time" @@ -43,12 +46,12 @@ func TestTransactionVerifierBoundsConcurrencyAndQueue(t *testing.T) { return nil }) defer verifier.closeAndWait() - if cap(verifier.jobs) != 2*workers { - t.Fatalf("queue capacity = %d, want %d", cap(verifier.jobs), 2*workers) + if got, want := cap(verifier.jobs), 1; got != want { + t.Fatalf("group queue capacity = %d, want %d", got, want) } done := make(chan error, 1) - go func() { done <- verifier.verifyBlock(verifierTestBlock(12)) }() + go func() { done <- verifier.verifyBlock(verifierTestBlock(24)) }() deadline := time.After(3 * time.Second) for active.Load() != workers { select { @@ -76,7 +79,7 @@ func TestTransactionVerifierBoundsConcurrencyAndQueue(t *testing.T) { } func TestTransactionVerifierReturnsLowestFailingIndex(t *testing.T) { - blk := verifierTestBlock(6) + blk := verifierTestBlock(24) lowErr := errors.New("low index failure") highErr := errors.New("high index failure") verifier := newTransactionVerifier(4, 8, func(tx *solana.Transaction) error { @@ -84,7 +87,7 @@ func TestTransactionVerifierReturnsLowestFailingIndex(t *testing.T) { case blk.Transactions[1]: time.Sleep(10 * time.Millisecond) return lowErr - case blk.Transactions[3]: + case blk.Transactions[9]: return highErr default: return nil @@ -132,8 +135,8 @@ func TestTransactionVerifierRejectsNilAtDeterministicIndex(t *testing.T) { // Every transaction in a block must be verified and joined, whatever the count. // Workers group transactions, so a count that divides badly into groups must -// not leave a remainder waiting for company: verifyBlockContext joins each -// wave with done.Wait(), and a stranded job would hang it forever. +// not leave a remainder waiting for company: every tail is dispatched as +// soon as it is available, without waiting for another request. // // A counting verifier is injected so the assertion is on what was actually // verified, not merely on returning without error. @@ -161,3 +164,290 @@ func TestTransactionVerifierVerifiesEveryTransactionForAwkwardCounts(t *testing. }) } } + +func TestTransactionVerifierRefillsWhileEarlierGroupIsBlocked(t *testing.T) { + blk := verifierTestBlock(24) + started := make(chan struct{}) + release := make(chan struct{}) + refilled := make(chan struct{}) + v := newTransactionVerifierWithBatchTarget(2, 16, 8, func(tx *solana.Transaction) error { + switch tx { + case blk.Transactions[0]: + close(started) + <-release + case blk.Transactions[16]: + close(refilled) + } + return nil + }) + defer v.closeAndWait() + defer close(release) + r, err := v.submitTransactions(context.Background(), blk.Transactions) + require.NoError(t, err) + waitSignal(t, started, "slow first group") + waitSignal(t, refilled, "rolling refill before first group finishes") + select { + case <-r.done: + t.Fatal("request finished without joining its blocked group") + default: + } +} + +func TestTransactionVerifierPartialTailStartsWithoutAnotherSubmission(t *testing.T) { + for _, target := range []int{4, 8} { + t.Run(fmt.Sprintf("target=%d", target), func(t *testing.T) { + seen := make(chan struct{}, 3) + v := newTransactionVerifierWithBatchTarget(2, 16, target, func(*solana.Transaction) error { + seen <- struct{}{} + return nil + }) + defer v.closeAndWait() + r, err := v.submitTransactions(context.Background(), verifierTestBlock(3).Transactions) + require.NoError(t, err) + for range 3 { + waitSignal(t, seen, "available partial batch transaction") + } + _, err = r.wait() + require.NoError(t, err) + require.False(t, r.finishedAt.IsZero()) + }) + } +} + +func TestTransactionVerifierLargeRequestDoesNotQueuePastSmallRequest(t *testing.T) { + large := verifierTestBlock(800) + small := verifierTestBlock(1) + started := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + var largeCalls atomic.Int32 + var callsBeforeSmall atomic.Int32 + v := newTransactionVerifierWithBatchTarget(1, 16, 8, func(tx *solana.Transaction) error { + if tx == small.Transactions[0] { + callsBeforeSmall.Store(largeCalls.Load()) + return nil + } + largeCalls.Add(1) + if tx == large.Transactions[0] { + close(started) + <-release + } + return nil + }) + defer v.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + largeRequest, err := v.submitTransactions(context.Background(), large.Transactions) + require.NoError(t, err) + waitSignal(t, started, "large request first group") + smallRequest, err := v.submitTransactions(context.Background(), small.Transactions) + require.NoError(t, err) + require.Eventually(t, func() bool { return len(v.jobs) == 1 }, 3*time.Second, time.Millisecond) + releaseOnce.Do(func() { close(release) }) + _, err = smallRequest.wait() + require.NoError(t, err) + require.Equal(t, int32(v.batchTarget*v.jobGroups), callsBeforeSmall.Load(), "large request may only stay one job ahead") + _, err = largeRequest.wait() + require.NoError(t, err) + require.Equal(t, int32(800), largeCalls.Load()) +} + +func TestTransactionVerifierAsyncAdmissionAppliesCancelableBackpressure(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var first sync.Once + v := newTransactionVerifierWithBatchTarget(1, 8, 8, func(*solana.Transaction) error { + first.Do(func() { close(started) }) + <-release + return nil + }) + defer v.closeAndWait() + defer close(release) + for range 2 { + _, err := v.submitTransactions(context.Background(), verifierTestBlock(1).Transactions) + require.NoError(t, err) + } + waitSignal(t, started, "occupied request slots") + require.Equal(t, cap(v.requests), len(v.requests)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + result := make(chan error, 1) + go func() { + _, err := v.submitTransactions(ctx, verifierTestBlock(1).Transactions) + result <- err + }() + select { + case err := <-result: + t.Fatalf("unbounded request admitted instead of waiting: %v", err) + case <-time.After(20 * time.Millisecond): + } + cancel() + select { + case err := <-result: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(3 * time.Second): + t.Fatal("request admission ignored cancellation") + } +} + +func TestTransactionVerificationWaitContextCancelsAndJoins(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + var releaseOnce sync.Once + var seen atomic.Int32 + v := newTransactionVerifierWithBatchTarget(1, 8, 8, func(*solana.Transaction) error { + if seen.Add(1) == 1 { + close(started) + <-release + } + return nil + }) + defer v.closeAndWait() + defer releaseOnce.Do(func() { close(release) }) + r, err := v.submitTransactions(context.Background(), verifierTestBlock(800).Transactions) + require.NoError(t, err) + waitSignal(t, started, "first admitted group") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + result := make(chan error, 1) + go func() { _, err := r.waitContext(ctx); result <- err }() + select { + case err := <-result: + t.Fatalf("wait returned while transactions still owned by worker: %v", err) + case <-time.After(20 * time.Millisecond): + } + releaseOnce.Do(func() { close(release) }) + select { + case err := <-result: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(3 * time.Second): + t.Fatal("canceled request did not join") + } + require.Equal(t, int32(8), seen.Load(), "cancellation must stop later group admission") +} + +func TestTransactionVerifierCloseRacesAdmissionWithoutStrandingRequests(t *testing.T) { + v := newTransactionVerifier(2, 16, func(*solana.Transaction) error { return nil }) + var callers sync.WaitGroup + for range 32 { + callers.Go(func() { + r, err := v.submitTransactions(context.Background(), verifierTestBlock(17).Transactions) + if err != nil { + if !errors.Is(err, errTransactionVerifierClosed) { + t.Errorf("submit error: %v", err) + } + return + } + _, err = r.wait() + if err != nil { + t.Errorf("admitted request error: %v", err) + } + }) + } + v.closeAndWait() + callers.Wait() + _, err := v.submitTransactions(context.Background(), verifierTestBlock(1).Transactions) + require.ErrorIs(t, err, errTransactionVerifierClosed) +} + +func verifierSignedTransactions(t *testing.T, count int) []*solana.Transaction { + t.Helper() + seed := make([]byte, ed25519.SeedSize) + seed[0] = 71 // Deterministic test-only key; never a validator identity. + key := ed25519.NewKeyFromSeed(seed) + var public solana.PublicKey + copy(public[:], key[32:]) + txs := make([]*solana.Transaction, count) + for i := range txs { + tx := &solana.Transaction{ + Message: solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1}, + AccountKeys: []solana.PublicKey{public}, + RecentBlockhash: solana.Hash{byte(i), byte(i >> 8)}, + }, + Signatures: make([]solana.Signature, 1), + } + message, err := tx.Message.MarshalBinary() + require.NoError(t, err) + copy(tx.Signatures[0][:], ed25519.Sign(key, message)) + txs[i] = tx + } + return txs +} + +func TestTransactionVerifierRejectsEveryInvalidSignatureLane(t *testing.T) { + v := newTransactionVerifier(2, 16, nil) + defer v.closeAndWait() + for invalid := range 8 { + t.Run(fmt.Sprintf("lane=%d", invalid), func(t *testing.T) { + txs := verifierSignedTransactions(t, 8) + txs[invalid].Signatures[0][13] ^= 0x40 + r, err := v.submitTransactions(context.Background(), txs) + require.NoError(t, err) + index, err := r.wait() + require.ErrorContains(t, err, "invalid signature") + require.Equal(t, invalid, index) + }) + } + r, err := v.submitTransactions(context.Background(), verifierSignedTransactions(t, 17)) + require.NoError(t, err) + _, err = r.wait() + require.NoError(t, err, "valid transactions must still verify after invalid lanes") +} + +func TestTransactionVerifierKeepsMultisignatureTransactionsIntactAcrossTargets(t *testing.T) { + counts := []int{2, 2, 1, 4, 9, 3, 2, 1} + txs := make([]*solana.Transaction, len(counts)) + for i, count := range counts { + keys := make([]ed25519.PrivateKey, count) + public := make([]solana.PublicKey, count) + for signer := range keys { + seed := make([]byte, ed25519.SeedSize) + seed[0], seed[1] = 83, byte(signer) + keys[signer] = ed25519.NewKeyFromSeed(seed) + copy(public[signer][:], keys[signer][32:]) + } + tx := &solana.Transaction{ + Message: solana.Message{ + Header: solana.MessageHeader{ + NumRequiredSignatures: uint8(count), + NumReadonlySignedAccounts: uint8(count - 1), + }, + AccountKeys: public, + RecentBlockhash: solana.Hash{byte(i)}, + }, + Signatures: make([]solana.Signature, count), + } + message, err := tx.Message.MarshalBinary() + require.NoError(t, err) + for signer, key := range keys { + copy(tx.Signatures[signer][:], ed25519.Sign(key, message)) + } + txs[i] = tx + } + for _, target := range []int{4, 8} { + for _, groups := range []int{1, 4, 8} { + t.Run(fmt.Sprintf("target=%d/groups=%d", target, groups), func(t *testing.T) { + v := newTransactionVerifierWithJobGroups(2, 16, target, groups, nil) + defer v.closeAndWait() + var large []*solana.Transaction + for range 40 { + large = append(large, txs...) + } + r, err := v.submitTransactions(context.Background(), large) + require.NoError(t, err) + _, err = r.wait() + require.NoError(t, err) + + // Corrupt a non-first signer after an oversized (nine-signature) + // transaction. Results must still map to the original tx index. + txs[6].Signatures[1][11] ^= 0x20 + r, err = v.submitTransactions(context.Background(), large) + require.NoError(t, err) + index, err := r.wait() + require.ErrorContains(t, err, "invalid signature") + require.Equal(t, 6, index) + txs[6].Signatures[1][11] ^= 0x20 + }) + } + } +} diff --git a/pkg/txstatus/message_identity.go b/pkg/txstatus/message_identity.go index 4d41f5450..4541a2a12 100644 --- a/pkg/txstatus/message_identity.go +++ b/pkg/txstatus/message_identity.go @@ -27,11 +27,18 @@ func TransactionMessageHash(tx *solana.Transaction) ([32]byte, error) { return messageHash, fmt.Errorf("serialize transaction message: %w", err) } + return HashCanonicalMessage(message), nil +} + +// HashCanonicalMessage hashes the exact canonical bytes used for transaction +// signature verification, including any message-version prefix. +func HashCanonicalMessage(message []byte) [32]byte { + var messageHash [32]byte hasher := blake3.New() _, _ = hasher.Write([]byte(transactionMessageHashDomain)) _, _ = hasher.Write(message) hasher.Sum(messageHash[:0]) - return messageHash, nil + return messageHash } // IdentityForTransaction captures both components needed for a status-cache diff --git a/pkg/txverify/message_identity.go b/pkg/txverify/message_identity.go new file mode 100644 index 000000000..4ba83bfc2 --- /dev/null +++ b/pkg/txverify/message_identity.go @@ -0,0 +1,25 @@ +package txverify + +import ( + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/gagliardetto/solana-go" +) + +// VerifiedMessageIdentity is an immutable result of signature verification. +// Its zero value is unusable. Signed message contents must remain immutable; +// as with Block's existing cache, arbitrary in-place edits require invalidation. +type VerifiedMessageIdentity struct { + transaction *solana.Transaction + version solana.MessageVersion + identity txstatus.TransactionMessageIdentity + verified bool +} + +// ForTransaction checks the binding without reserializing. Address-table +// resolution is allowed because it does not change the canonical message. +func (v VerifiedMessageIdentity) ForTransaction(tx *solana.Transaction) (txstatus.TransactionMessageIdentity, bool) { + if !v.verified || tx == nil || tx != v.transaction || tx.Message.GetVersion() != v.version || tx.Message.RecentBlockhash != v.identity.RecentBlockhash { + return txstatus.TransactionMessageIdentity{}, false + } + return v.identity, true +} diff --git a/pkg/txverify/message_identity_test.go b/pkg/txverify/message_identity_test.go new file mode 100644 index 000000000..29d265497 --- /dev/null +++ b/pkg/txverify/message_identity_test.go @@ -0,0 +1,70 @@ +package txverify + +import ( + "crypto/ed25519" + "encoding/hex" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/txstatus" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func identitySignedTransaction(t *testing.T, version solana.MessageVersion) *solana.Transaction { + t.Helper() + key := ed25519.NewKeyFromSeed(make([]byte, 32)) + tx := &solana.Transaction{Message: solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1, NumReadonlyUnsignedAccounts: 1}, + AccountKeys: []solana.PublicKey{solana.PublicKeyFromBytes(key.Public().(ed25519.PublicKey)), {2}}, + RecentBlockhash: solana.Hash{3}, + Instructions: []solana.CompiledInstruction{{ProgramIDIndex: 1, Accounts: []uint16{0}, Data: []byte{4}}}, + }} + _, err := tx.Message.SetVersion(version) + require.NoError(t, err) + msg, err := MessageBytes(tx) + require.NoError(t, err) + tx.Signatures = []solana.Signature{solana.SignatureFromBytes(ed25519.Sign(key, msg))} + return tx +} + +func TestVerifiedMessageIdentityCanonicalVersionsAndFailures(t *testing.T) { + wire, err := hex.DecodeString(rustV1FeeHeapTransaction) + require.NoError(t, err) + v1, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + txs := []*solana.Transaction{identitySignedTransaction(t, solana.MessageVersionLegacy), identitySignedTransaction(t, solana.MessageVersionV0), v1} + bad := identitySignedTransaction(t, solana.MessageVersionLegacy) + bad.Signatures[0][0] ^= 1 + txs = append(txs, bad, nil) + errs := make([]error, len(txs)) + ids := make([]VerifiedMessageIdentity, len(txs)) + var verifier BatchVerifier + verifier.VerifyWithMessageIdentities(txs, errs, ids) + for i := range txs { + id, ok := ids[i].ForTransaction(txs[i]) + if i >= 3 { + require.Error(t, errs[i]) + require.False(t, ok) + continue + } + require.NoError(t, errs[i]) + require.True(t, ok) + want, err := txstatus.IdentityForTransaction(txs[i]) + require.NoError(t, err) + require.Equal(t, want, id) + } + // Scratch reuse cannot invalidate a prior successful request, and reusing + // an output lane for failure must not leave a usable old identity behind. + saved := ids[0] + verifier.VerifyWithMessageIdentities([]*solana.Transaction{bad}, errs[:1], ids[:1]) + _, ok := saved.ForTransaction(txs[0]) + require.True(t, ok) + _, ok = ids[0].ForTransaction(txs[0]) + require.False(t, ok) + copyTx := *txs[0] + _, ok = saved.ForTransaction(©Tx) + require.False(t, ok) + txs[0].Message.RecentBlockhash[0] ^= 1 + _, ok = saved.ForTransaction(txs[0]) + require.False(t, ok) +} diff --git a/pkg/txverify/txverify.go b/pkg/txverify/txverify.go index 7221399c4..af58b9cb5 100644 --- a/pkg/txverify/txverify.go +++ b/pkg/txverify/txverify.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/Overclock-Validator/mithril/pkg/sigverify" + "github.com/Overclock-Validator/mithril/pkg/txstatus" "github.com/gagliardetto/solana-go" ) @@ -222,6 +223,21 @@ type BatchVerifier struct { // Every transaction gets an independent verdict: one bad transaction does not // mask the others, so a caller can report precisely which one failed. func (v *BatchVerifier) Verify(txs []*solana.Transaction, errs []error) { + v.verify(txs, errs, nil) +} + +// VerifyWithMessageIdentities additionally retains identities derived from the +// same canonical bytes used by signature verification. Only successful verdicts +// produce usable identities. The output is caller-owned, not verifier scratch. +func (v *BatchVerifier) VerifyWithMessageIdentities(txs []*solana.Transaction, errs []error, identities []VerifiedMessageIdentity) { + if len(identities) != len(txs) { + panic("txverify: identities and txs length mismatch") + } + clear(identities) + v.verify(txs, errs, identities) +} + +func (v *BatchVerifier) verify(txs []*solana.Transaction, errs []error, identities []VerifiedMessageIdentity) { if len(errs) != len(txs) { panic("txverify: errs and txs length mismatch") } @@ -239,6 +255,16 @@ func (v *BatchVerifier) Verify(txs []*solana.Transaction, errs []error) { v.signers = append(v.signers, nil) continue } + if identities != nil { + identities[i] = VerifiedMessageIdentity{ + transaction: tx, + version: tx.Message.GetVersion(), + identity: txstatus.TransactionMessageIdentity{ + MessageHash: txstatus.HashCanonicalMessage(msg), + RecentBlockhash: tx.Message.RecentBlockhash, + }, + } + } for j := range tx.Signatures { v.batch.Add((*[32]byte)(&signers[j]), msg, tx.Signatures[j][:]) } @@ -247,6 +273,9 @@ func (v *BatchVerifier) Verify(txs []*solana.Transaction, errs []error) { } if v.batch.Verify() { + for i := range identities { + identities[i].verified = errs[i] == nil + } return } @@ -257,6 +286,9 @@ func (v *BatchVerifier) Verify(txs []*solana.Transaction, errs []error) { errs[i] = fmt.Errorf("invalid signature by %s", v.signers[i][j]) } } + if identities != nil { + identities[i].verified = errs[i] == nil + } lane += count } }