diff --git a/cmd/repair-sim/main.go b/cmd/repair-sim/main.go new file mode 100644 index 000000000..5eebf05e9 --- /dev/null +++ b/cmd/repair-sim/main.go @@ -0,0 +1,137 @@ +// repair-sim runs deterministic, single-node Turbine repair scenarios. +package main + +import ( + "encoding/json" + "flag" + "fmt" + "os" + "os/exec" + "runtime" + "strings" + "time" + + "github.com/Overclock-Validator/mithril/pkg/turbine/repairsim" +) + +type environment struct { + GoVersion string `json:"go_version"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + CPU string `json:"cpu"` +} + +type report struct { + Environment environment `json:"environment"` + Ledger repairsim.LedgerConfig `json:"ledger"` + Network repairsim.Config `json:"network"` + LedgerGenerationWall time.Duration `json:"ledger_generation_wall_ns"` + Result repairsim.Result `json:"result"` +} + +func main() { + var ( + scenarioFlag = flag.String("scenario", string(repairsim.ScenarioNearTip), "near-tip or deep-catchup") + slots = flag.Int("slots", 200, "number of deterministic slots") + fecSets = flag.Int("fec-sets", 4, "FEC sets generated per slot") + entries = flag.Int("entries", 0, "entries per slot (0 derives an exact FEC count)") + seed = flag.Int64("seed", 1, "deterministic content and network seed") + availability = flag.String("availability", "", "complete, near-loss, sparse, or mixed") + repair = flag.Bool("repair", true, "enable repair requests") + latency = flag.Duration("repair-latency", 20*time.Millisecond, "synthetic one-way response latency") + jitter = flag.Duration("repair-jitter", 2*time.Millisecond, "deterministic +/- response jitter") + loss = flag.Float64("packet-loss", 0, "repair response loss probability [0,1]") + duplicates = flag.Float64("duplicates", 0.02, "duplicate response probability [0,1]") + bandwidth = flag.Int64("repair-bandwidth", 100*1024*1024, "synthetic repair bytes/sec (0 is unlimited)") + concurrent = flag.Int("max-concurrent", 256, "maximum outstanding repair shreds") + corrupt = flag.Int("corrupt-responses", 0, "corrupt the first N repair responses") + naturalLate = flag.Bool("natural-late", true, "schedule selected late live shreds during repair") + spoolDir = flag.String("spool-dir", "", "persistent shred-spool directory (empty uses a temporary directory)") + cpuLabel = flag.String("cpu-label", "", "explicit CPU label when platform discovery is unavailable") + output = flag.String("output", "", "write JSON to this file instead of stdout") + includeTrace = flag.Bool("trace", true, "include the logical event trace in JSON") + ) + flag.Parse() + + scenario := repairsim.Scenario(*scenarioFlag) + network := repairsim.DefaultConfig(scenario) + network.Availability = repairsim.Availability(*availability) + network.RepairEnabled = *repair + network.RepairLatency = *latency + network.RepairJitter = *jitter + network.PacketLoss = *loss + network.DuplicateProbability = *duplicates + network.BandwidthBytesPerSec = *bandwidth + network.MaxConcurrent = *concurrent + network.CorruptResponses = *corrupt + network.NaturalLateShreds = *naturalLate + network.CollectTrace = *includeTrace + network.Seed = *seed + network.SpoolDir = *spoolDir + if network.Availability == "" { + network.Availability = repairsim.DefaultConfig(scenario).Availability + } + + ledgerCfg := repairsim.LedgerConfig{ + StartSlot: 10_000, + Slots: *slots, + FECsPerSlot: *fecSets, + EntriesPerSlot: *entries, + Seed: *seed, + ShredVersion: 1, + ReferenceTick: 63, + } + started := time.Now() + ledger, err := repairsim.GenerateLedger(ledgerCfg) + if err != nil { + fatalf("generate ledger: %v", err) + } + generationWall := time.Since(started) + result, err := repairsim.Run(ledger, network) + if err != nil { + fatalf("run simulation: %v", err) + } + cpu := *cpuLabel + if cpu == "" { + cpu = cpuModel() + } + report := report{ + Environment: environment{GoVersion: runtime.Version(), GOOS: runtime.GOOS, GOARCH: runtime.GOARCH, CPU: cpu}, + Ledger: ledger.Config, Network: network, LedgerGenerationWall: generationWall, Result: result, + } + encoded, err := json.MarshalIndent(report, "", " ") + if err != nil { + fatalf("marshal report: %v", err) + } + encoded = append(encoded, '\n') + if *output == "" { + _, _ = os.Stdout.Write(encoded) + return + } + if err := os.WriteFile(*output, encoded, 0o644); err != nil { + fatalf("write %s: %v", *output, err) + } +} + +func cpuModel() string { + if runtime.GOOS == "linux" { + if data, err := os.ReadFile("/proc/cpuinfo"); err == nil { + for _, line := range strings.Split(string(data), "\n") { + if key, value, ok := strings.Cut(line, ":"); ok && strings.TrimSpace(key) == "model name" { + return strings.TrimSpace(value) + } + } + } + } + if runtime.GOOS == "darwin" { + if out, err := exec.Command("sysctl", "-n", "machdep.cpu.brand_string").Output(); err == nil { + return strings.TrimSpace(string(out)) + } + } + return "unknown" +} + +func fatalf(format string, args ...any) { + _, _ = fmt.Fprintf(os.Stderr, format+"\n", args...) + os.Exit(1) +} diff --git a/docs/erasure_recovery_experiments.md b/docs/erasure_recovery_experiments.md new file mode 100644 index 000000000..dc2ed2d25 --- /dev/null +++ b/docs/erasure_recovery_experiments.md @@ -0,0 +1,185 @@ +# Erasure recovery experiments + +Status: `SlotAssembler` dispatches the fixed 32+32, exactly-one-missing-data +case to the direct recovery path. Reduced multi-missing and all-coding paths +remain experimental and are not used by production dispatch. + +The end-to-end deterministic harness that drives production repair selection, +assembly, storage, and completion is documented in [repair_sim.md](repair_sim.md). + +This document separates two repair regimes that have different objectives. It +also records the fixed 32 data + 32 coding Reed-Solomon contract used by the +synthetic implementation in `pkg/turbine/internal/rsrecover`. + +## Regimes + +Near-tip repair minimizes the time until replay receives a particular blocking +data shred. Its primary candidate is direct recovery when exactly one data +shred is absent and at least one coding shred is present. + +Catch-up repair minimizes useful recovered-data time across many incomplete FEC +sets. Its candidate constructs only the reduced system induced by the missing +data columns and produces only missing data outputs. + +Slot age alone should not select the regime. A future policy experiment should +consume observed Turbine progress: + +- number and fraction of incomplete FEC sets; +- missing data shreds per FEC set; +- whether new shreds are still arriving; +- time or scheduling intervals since the last useful arrival; +- number of FEC sets that have crossed the recovery threshold; +- data-heavy versus coding-heavy availability. + +A progressing slot with one or two holes remains a near-tip workload even if it +is not the newest slot. A stalled slot with many incomplete FEC sets is a +catch-up workload even if wall-clock age is modest. Any eventual selector needs +hysteresis so bursty arrivals cannot oscillate the algorithm on every packet. + +## Matrix contract + +The experiment uses the systematic generator over `GF(256)/0x11d`: + +```text +V[x,j] = x^j +A = V[0:32,0:32] +G = V * A^-1 +G = [I_32; C] +``` + +For the fixed 32+32 shape, exhaustive scalar tests confirm all 1,024 entries: + +```text +C[r,c] = 0xa5 / (0x20 xor r xor c) +``` + +They also confirm `C*C=I`. Recovery tests remain differential against +`github.com/klauspost/reedsolomon`; the closed form is not the sole oracle. + +## Candidates + +### Near tip: direct one-data recovery + +For missing data position `m` and available coding position `r`: + +```text +D_m = C[r,m]^-1 * P_r + + sum(i != m, C[r,m]^-1 * C[r,i] * D_i) +``` + +This prepares one 32-source coefficient row and writes one destination. It does +not construct or invert a general 32x32 matrix. + +Production uses a process-wide table containing every missing-data and +coding-row combination. This removes per-call plan construction while keeping +the same equation. The table is exhaustively differential-tested against the +general decoder across all 32 x 32 combinations. + +### Catch up: reduced missing-data system + +For missing data columns `M` and selected coding rows `R`, substitute every +known data shard and solve: + +```text +B[i,j] = C[R[i],M[j]] +B * D_M = adjusted_coding_rows +``` + +The experiment uses the Cauchy closed form to construct `B^-1` in `O(m^2)`, +expands direct rows over 32 selected sources, and proves each row satisfies the +requested systematic generator row before byte processing. An independently +implemented Gauss-Jordan inverse is the setup fallback. + +The byte kernel is intentionally portable and uses `reedsolomon.LowLevel`. +This isolates algorithm and plan costs; it is not evidence that a portable +kernel will beat the dependency's generated AVX2/GFNI kernels on amd64. + +### Catch up edge: all coding rows + +When all data rows are missing and every coding row is present, `C*C=I` means +the existing optimized encoder can apply `C` to the coding rows and recover the +data directly. This is kept as a separate synthetic arm. It is simpler than a +general decoder, but a cached reduced-system plan may still have a faster byte +kernel; hardware decides between them. + +## Synthetic coverage + +The tests cover: + +- every missing-data position with every coding-row choice for the direct path; +- every pair of missing data positions; +- deterministic mixed patterns at 2, 4, 8, 16, 24, and 32 missing data shreds; +- exactly-threshold and one-below-threshold availability; +- changed availability between setup and execution; +- destination failure atomicity; +- coefficient mutation detection; +- Cauchy inverses against independent Gauss-Jordan inversion; +- recovered bytes against the existing general decoder. + +Run: + +```bash +go test ./pkg/turbine/internal/rsrecover +go test -run '^$' \ + -bench '^(BenchmarkRecoverOneData|BenchmarkRecoverDataSubset)$' \ + -benchmem -benchtime=2s -count=6 \ + ./pkg/turbine/internal/rsrecover +``` + +Benchmark result interpretation must keep these cases separate: + +- `prepare`: cold or changing erasure pattern; +- `execute`: prepared/repeated pattern; +- `prepare-and-execute`: first useful output for a new pattern; +- general cache off: existing decoder with changing pattern cost; +- general cache on: existing decoder after the inversion is cached. + +No production dispatch threshold should be chosen from an Apple benchmark. +Final crossover decisions require the pinned amd64 target and synthetic arrival +traces for progressing, stalled, and bursty slots. + +## Zen 5 production gate + +The direct one-data path was measured on a Ryzen 7 9700X with Go 1.26.4, +`GOMAXPROCS=1`, and one pinned physical core. Medians below are from seven +sequential one-second samples unless otherwise noted. + +| Benchmark | General path | Direct one-data path | Change | +| --- | ---: | ---: | ---: | +| one-missing `SlotAssembler` boundary | 10.73 us/FEC | 2.82 us/FEC | -73.7% (3.8x) | +| near-tip repair simulation | 3.0295 ms/op | 2.8659 ms/op | -5.40% | +| deep-mixed repair simulation | 4.0343 ms/op | 4.0240 ms/op | -0.26% | +| deep-sparse repair simulation | 10.8489 ms/op | 10.8327 ms/op | -0.15% | + +The production dispatch is intentionally narrow. The deep scenarios do not +enter it and remain effectively neutral, while the near-tip workload benefits +from repeated exactly-one-missing recoveries. The one-missing boundary also +dropped from 144 to 5 allocations per operation. + +## Preliminary Apple M4 Pro diagnostic + +These single-sample medians use 987-byte shards and exist only to reject or +retain candidates before the amd64 gate. Times are microseconds per FEC set. + +| Missing data | Specialized first use | Specialized prepared | General uncached | General cached | +|---:|---:|---:|---:|---:| +| 1 | 1.97 | 1.07 | 8.36 | 2.31 | +| 2 | 4.19 | 2.10 | 10.78 | 3.39 | +| 4 | 8.84 | 4.19 | 17.00 | 5.91 | +| 8 | 18.94 | 8.24 | 25.54 | 10.06 | +| 16 | 39.90 | 16.44 | 44.93 | 19.78 | +| 24 | 62.14 | 24.71 | 63.43 | 28.55 | +| 32 | 79.50 | 32.76 | 89.53 | 37.53 | + +The all-coding involution arm measured approximately 36.5 microseconds, +compared with 82.7 microseconds for an uncached general decode and 37.5 +microseconds for its cached form. Its main possible value is avoiding plan +setup; the prepared reduced-system byte path was faster on this machine. + +The current interpretation is deliberately conditional: + +- direct one-data recovery is strong enough to require an amd64 prototype; +- reduced-system first use wins through most of the tested range, but the + 24-missing crossover is within noise on this machine; +- prepared reduced-system execution wins at every tested width; +- none of these figures establishes a production policy or Zen 5 result. diff --git a/docs/repair_sim.md b/docs/repair_sim.md new file mode 100644 index 000000000..b0240f015 --- /dev/null +++ b/docs/repair_sim.md @@ -0,0 +1,153 @@ +# Deterministic repair simulation + +`cmd/repair-sim` is a single-process harness for measuring the local path from +an incomplete slot to a block that is available to replay. It exists because +near-tip repair and deep catch-up optimize different outcomes: + +- near the tip, latency of the first replay-blocking slot matters; +- during catch-up, sustained useful data and completed slots per second matter. + +The first implementation deliberately stops before UDP and transaction +execution. It establishes a deterministic, correctness-checked baseline before +network realism or alternative scheduling policies are introduced. + +## What is real and what is simulated + +| Stage | Implementation | +| --- | --- | +| block-component serialization | production `turbine.MarshalBlockComponent` | +| 32+32 FEC generation and Merkle signing | production `turbine.Shredder` | +| missing-shred selection | production `SlotAssembler.RepairRequests` | +| packet parsing and Merkle/signature validation | production `ParseShred` and `ShredSignatureVerifier` (the receiver's per-root cache) | +| verified-shred insertion | production `ShredSpool` | +| threshold detection and Reed-Solomon recovery | production `SlotAssembler.AddShredFrom` | +| component decode and transaction-signature gate | production slot completion path | +| remote peer, latency, jitter, loss, duplication, bandwidth | deterministic in-process simulator | +| replay notification | block emission is recorded as “offered to replay” | +| transaction execution | not run in this version | + +Synthetic entries are valid Alpenglow entry-batch components but contain no +transactions. This isolates shred/FEC/repair/storage costs; it is not a replay +execution benchmark. + +## Scenarios + +### Near tip + +`near-loss` alternates two useful patterns across FEC sets: + +- 30 data + 1 coding shred: one repair response crosses the threshold and + reconstructs the other missing data shred; +- 31 data shreds: the one missing data shred must be fetched directly. + +Selected omitted shreds can also arrive through the simulated live path while +a repair response is outstanding. This measures cancellation/late-response +behavior without changing production scheduling. + +Disabling repair stops request scheduling but still delivers natural late live +shreds. The run ends when all slots complete or those live arrivals are +exhausted; incomplete slots are reported without a repair-stall error. + +### Deep catch-up + +`mixed` begins every FEC set with 16 data + 15 coding shreds. One fetched data +shred crosses the threshold and reconstructs the remaining 15. + +`sparse` begins every FEC set with two data shreds and no coding layout. The +ordinary repair interface serves data shreds only, so nearly all missing data +must arrive over the simulated network. Comparing `mixed` with `sparse` +quantifies the network work avoided by already-held coding shreds. + +## Commands + +```sh +go test ./pkg/turbine/repairsim + +go run ./cmd/repair-sim \ + -scenario=near-tip \ + -slots=200 \ + -fec-sets=4 \ + -seed=1 \ + -cpu-label='Ryzen 7 9700X' \ + -output=/tmp/repair-near.json + +go run ./cmd/repair-sim \ + -scenario=deep-catchup \ + -availability=mixed \ + -slots=1000 \ + -fec-sets=4 \ + -seed=1 \ + -repair-latency=20ms \ + -repair-bandwidth=104857600 \ + -output=/tmp/repair-deep-mixed.json + +go run ./cmd/repair-sim \ + -scenario=deep-catchup \ + -availability=sparse \ + -slots=1000 \ + -fec-sets=4 \ + -seed=1 \ + -repair-latency=20ms \ + -repair-bandwidth=104857600 \ + -output=/tmp/repair-deep-sparse.json + +go test ./pkg/turbine/repairsim \ + -run '^$' -bench '^BenchmarkScenarios$' -benchmem -count=5 +``` + +Logical trace timestamps are deterministic. `wall_elapsed_ns`, allocations, +and `stage_cpu_ns` are actual local measurements and therefore are not expected +to be byte-identical across runs. + +## Correctness gates + +The current tests require: + +- exact requested FEC-set counts from authentic generated shreds; +- Merkle/signature validity for every canonical packet; +- no completed slot when loss is present and repair is disabled; +- complete, canonical entry streams after threshold recovery; +- a complete `ShredSpool` journal record before replay admission; +- rejection and retry of a corrupted repair response; +- deterministic logical traces for identical seeds; +- late repair responses to leave a completed block unchanged. + +The Turbine package also retains focused byte-for-byte Reed-Solomon recovery +tests. The simulator verifies the stronger end-to-end consequence: recovered +shreds must decode to the exact canonical entry sequence and pass the normal +completion gates. + +## Generator compatibility finding + +Building this harness exposed a multi-FEC component bug: the local generator +set `DATA_COMPLETE_SHRED` at every FEC boundary. The decoder correctly treats +that flag as the end of one serialized component, so a component spanning more +than one FEC set was truncated and failed to decode. The generator now follows +Agave's ordering: construct every FEC set, then mark only the final data shred +of the component complete (or last-in-slot). A regression test round-trips one +1,300-entry component across multiple FEC sets. + +## Future mode selection + +No production mode switch is added here. A later policy experiment should use +both replay distance and observed Turbine usefulness, with hysteresis: + +- stay in near-tip mode while the replay gap is small and Turbine supplies a + high fraction of useful shreds before repair deadlines; +- enter catch-up mode only when the replay-blocking gap is sustained and live + Turbine delivery is insufficient to approach FEC thresholds; +- return to near-tip mode only after both the gap and repair backlog fall below + lower thresholds. + +That signal is preferable to slot distance alone: a node may be numerically +close to the tip while receiving too few live shreds, or far behind while its +local spool already holds most FEC thresholds. + +## Next steps + +1. Add shallow catch-up and whole-block-pressure configurations. +2. Add a loopback-UDP transport without replacing the deterministic mode. +3. Expose internal FEC start/finish timing through opt-in instrumentation. +4. Run replay execution against a reusable synthetic bank fixture. +5. Compare ordinary requests with explicit test-only threshold-acquisition and + earliest-blocked-slot policies. diff --git a/docs/results/producer-batch/2026-09-06-zen5/README.md b/docs/results/producer-batch/2026-09-06-zen5/README.md new file mode 100644 index 000000000..fcf3d9f41 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/README.md @@ -0,0 +1,101 @@ +# Producer batching and shred-generation benchmarks + +Measured September 6, 2026 on an AMD Ryzen 7 9700X (Zen 5), Linux amd64, +Go 1.26.4, `GOAMD64=v1`, `CGO_ENABLED=0`. Each process used `GOMAXPROCS=1` +and was pinned to logical CPU 6. SMT and frequency boost were enabled on the +shared host. These are serial producer workloads on one logical CPU. + +## Comparison with current alpenglow-dev + +A subsequent comparison uses dev head `7e4e8af1`, which already has incremental +size accounting and one-FEC batching. The PR is **2.60x faster** for maximum-size +transactions and **2.10x faster** for small transfers in the serial producer +workload. The maximum-size comparison uses three slots on both sides to fit +dev's slot limits. See [the branch-head comparison](alpenglow-dev-head/README.md) +for raw samples, matched-batch controls, and full scope. + +The tables below retain the earlier comparison against PR commit `84776893`; +they do not use the current dev head as their baseline. + +## What changed + +The reviewed branch already increased the batch target to 61,632 bytes. +Its EntryBuilder reserialized all pending transactions on every append to +estimate the next batch size, making batch construction quadratic in the +number of transactions per batch. The follow-up changes: + +1. Measure each appended transaction once and accumulate its encoded size. +2. Return that tracked size on flush, eliminating serialization whose output + was immediately discarded. +3. Carry generated packets and every FEC root directly into BroadcastSession, + eliminating owning shred parsing, payload copies, and root reconstruction. + +## Complete-workload results + +Medians of five samples per version, one full 50,000-transaction workload per +sample. Versions ran sequentially with order reversed on alternating rounds. + +| Workload | Original | Incremental accounting | Final | Overall speedup | +| --- | ---: | ---: | ---: | ---: | +| 1,232-byte transactions, two slots of 25k | 2,114.572 ms | 384.560 ms | 285.456 ms | 7.41x | +| 215-byte transfers, one slot of 50k | 2,810.088 ms | 97.931 ms | 75.332 ms | 37.30x | + +The maximum-size workload is split to fit the default data-shred budget: +32,768 data shreds per generated slot. The benchmark includes entry building, +serialization, erasure coding, Merkle proofs, shred signing, and session/block +bookkeeping, including final flushes. The packet sink only counts packets. +Fixture construction and validation, execution, admission signature checks, +routing, UDP, and receiver processing are outside timing. These are CPU-stage +measurements, not executed-block latency or whole-validator throughput. + +Detailed methods, allocations, packet counts, and raw samples: + +- [Maximum-size transactions](max-size/README.md) +- [Small transfer transactions](small-transfer/README.md) +- [Steady-state per-transaction measurements](steady-state.md), which compare + the intermediate and final implementations only + +## Baselines and reproduction + +**Original** uses the producer implementation at reviewed branch commit +`847768930cde10b6c885e4a82648d192b34098a9`. **Intermediate** adds incremental +transaction-size accounting. **Final** also removes the flush serialization +pass and uses generated packets/FEC roots directly in BroadcastSession. +All three use the same 61,632-byte target and the same workload code. +The table measures follow-up fixes against the already-expanded batching +implementation; it is **not a comparison with the PR base branch**, whose +batch target was only 1,926 bytes. + +The timed production files in the original snapshot match that reviewed +commit byte-for-byte. The intermediate snapshot changes only EntryBuilder +in that path. The final snapshot's timed production files match this PR. +The benchmark files checked in here are the measured sources with gofmt +formatting; no workload or timing logic was changed. + +To run the final benchmarks on Linux: + +```sh +CGO_ENABLED=0 GOAMD64=v1 go test -c -o /tmp/mithril-producer.test ./pkg/blockprod +GOMAXPROCS=1 taskset -c 6 /tmp/mithril-producer.test \ + -test.run '^TestMaxSizeProducerFixture$' +GOMAXPROCS=1 taskset -c 6 /tmp/mithril-producer.test \ + -test.run '^$' -test.bench '^(BenchmarkProducerBlock50k|BenchmarkProducer50kMaxSize)$' \ + -test.benchmem -test.benchtime=1x -test.count=5 +``` + +For an A/B reproduction, create separate clean checkouts at the reviewed +commit and at this PR's final commit. Copy these identical files from the +final checkout into each baseline: + +- `pkg/blockprod/entry_bench_test.go` (shared packet-counting sink) +- `pkg/blockprod/producer_block_bench_test.go` +- `pkg/blockprod/producer_maxsize_bench_test.go` + +For the intermediate checkout, additionally apply +[incremental-accounting.patch](incremental-accounting.patch) from its repo root +with `git apply --unidiff-zero `. +Build each test binary with the same toolchain and environment. Run each +workload separately with `-test.count=1`, alternating original/intermediate/final +and final/intermediate/original for five rounds, as in the recorded run. +Report the median ns/op, B/op, and allocs/op across each version's five samples. +Select an available CPU on your host in place of CPU 6 if necessary. diff --git a/docs/results/producer-batch/2026-09-06-zen5/after.txt b/docs/results/producer-batch/2026-09-06-zen5/after.txt new file mode 100644 index 000000000..367748990 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/after.txt @@ -0,0 +1,40 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 2560165 470.6 ns/op 990 B/op 7 allocs/op +BenchmarkEntryBuilder/append-and-shred 762813 1556 ns/op 3468 B/op 15 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 835798 1407 ns/op 2817 B/op 14 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 2553692 466.1 ns/op 990 B/op 7 allocs/op +BenchmarkEntryBuilder/append-and-shred 762499 1548 ns/op 3468 B/op 15 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 814294 1409 ns/op 2818 B/op 14 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 2572012 465.7 ns/op 990 B/op 7 allocs/op +BenchmarkEntryBuilder/append-and-shred 766179 1545 ns/op 3468 B/op 15 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 837370 1411 ns/op 2817 B/op 14 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 2580415 465.6 ns/op 990 B/op 7 allocs/op +BenchmarkEntryBuilder/append-and-shred 753132 1541 ns/op 3468 B/op 15 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 827732 1405 ns/op 2818 B/op 14 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 2590515 467.4 ns/op 990 B/op 7 allocs/op +BenchmarkEntryBuilder/append-and-shred 756445 1544 ns/op 3467 B/op 15 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 835386 1404 ns/op 2817 B/op 14 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/README.md b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/README.md new file mode 100644 index 000000000..8cb509587 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/README.md @@ -0,0 +1,126 @@ +# Producer comparison with alpenglow-dev head + +Measured September 6, 2026 on an AMD Ryzen 7 9700X (Zen 5), Linux amd64, +Go 1.26.4, `CGO_ENABLED=0`, `GOAMD64=v1`, `GOMAXPROCS=1`, pinned to logical +CPU 6. The host was shared, with SMT and boost enabled; no running service +or CPU-affinity setting was changed. Medians of five alternating one-iteration +samples per arm. Both test binaries were built on this host with the same +compiler and identical benchmark source. + +## Exact baselines + +- `alpenglow-dev`: `7e4e8af115a27aa15f3401d59f3be1f0d79a4c1b`, the fetched head + at measurement time. It already has incremental size accounting, a + 30,816-byte (one FEC) batch target, slot entry-byte limits, and an asynchronous + shred worker. +- PR #259 producer: `a598cd0508cab1788c688b8ce667c6c810e1f2d5`, using its own + 61,632-byte (two FEC) default. +- Control: the same PR code, setting `Limits.MaxBatchBytes = 30,816` only in + the benchmark, to match the dev head's target. + +Production source was unchanged in both snapshots. The standalone benchmark +was copied into each snapshot's `pkg/blockprod` directory. This compares the +two heads' serial producer CPU stages; it does not measure a rebased or ported +version of this PR on top of dev. The PR's target branch remains unchanged. + +## Results: 50,000 transactions + +| Workload | alpenglow-dev default | PR default | Speedup | Time reduction | +| --- | ---: | ---: | ---: | ---: | +| 1,232-byte transactions, three slots | 734.052 ms | 282.296 ms | 2.60x | 61.54% | +| 215-byte transfers, one slot | 150.339 ms | 71.442 ms | 2.10x | 52.48% | + +With both implementations using the **same 30,816-byte target**: + +| Workload | alpenglow-dev | PR at one FEC | Speedup | +| --- | ---: | ---: | ---: | +| 1,232-byte transactions, three slots | 734.052 ms | 288.378 ms | 2.55x | +| 215-byte transfers, one slot | 150.339 ms | 72.424 ms | 2.08x | + +The measured benefit largely survives with a one-FEC target. Moving the PR +from one to two FECs reduces elapsed time by about 2.1% for maximum-size +transactions and 1.4% for small transfers in these samples. This does not +establish the best batch target for live latency or pipelined throughput. + +The earlier 7.41x/37.30x figures compare follow-up fixes against the older PR +implementation, before incremental size accounting. They are historical +within-branch results, not improvements over the current dev head. + +## Workload and limits + +The same pool of 512 signed, parsed transactions is reused. Small transfers +are exactly 215 bytes. Maximum-size transactions are legacy system transfers +plus a 981-byte UTF-8 memo, exactly 1,232 bytes. All fixtures pass structural +sanitization, signature verification, and canonical byte round trips before +timing. + +Small transfers use one 50,000-transaction generation pass. Maximum-size +transactions use **16,667 + 16,667 + 16,666 transactions across three slots** +on every arm. This fits both dev's entry-byte bound (20 MiB minus the reserved +48-byte ending tick) and the default 32,768 data-shred limit. The harness +asserts both limits, transaction/batch/packet counts, and a nonzero block ID. +It does not run the bank's admission or execution paths. + +The original two-slot maximum-size workload cannot be reused unchanged for +the dev baseline: 25,000 such transactions exceed its entry-byte limit, and +its one-FEC batching would also exceed the data-shred limit. This report uses +three slots on both sides; its timings should be compared within this table. + +Included: EntryBuilder append/flush, entry serialization, erasure coding, +Merkle proofs, shred signing, headers/footers/ending ticks, chained roots, and +block-ID bookkeeping. The sink only counts packets. Excluded: fixture setup, +transaction execution, admission signature verification, scheduler/reservation +policy, asynchronous shred-worker queueing/overlap, routing, UDP, and receiver +work. **This is not whole-validator throughput or executed-block latency.** + +## Work and allocation counts + +| Workload / implementation | Entry batches | Packets, data + coding | Max data shreds / slot | Allocated bytes / workload | Allocations / workload | +| --- | ---: | ---: | ---: | ---: | ---: | +| Maximum / dev | 2,085 | 134,016 | 22,336 | 2,129,429,048 | 3,844,267 | +| Maximum / PR default | 1,023 | 131,328 | 21,888 | 708,002,592 | 974,514 | +| Maximum / PR one FEC | 2,085 | 134,016 | 22,336 | 712,676,760 | 997,857 | +| Small / dev | 350 | 22,592 | 11,296 | 374,665,624 | 1,215,632 | +| Small / PR default | 175 | 22,592 | 11,296 | 141,461,632 | 731,204 | +| Small / PR one FEC | 350 | 22,592 | 11,296 | 141,777,944 | 735,400 | + +## Reproduction and raw data + +Copy [producer_branch_bench_test.go](../../../../../pkg/blockprod/producer_branch_bench_test.go) +into `pkg/blockprod` in clean checkouts of the exact dev and PR commits above. +This file contains its own fixture and sink helpers and can be used without +the PR's other benchmark files. + +Build each binary on Linux: + +```sh +CGO_ENABLED=0 GOAMD64=v1 go test -c -o /tmp/producer-dev.test ./pkg/blockprod +# Repeat in the PR checkout, changing the output to /tmp/producer-pr.test. +GOMAXPROCS=1 taskset -c 6 /tmp/producer-dev.test \ + -test.run '^TestProducerBranchComparisonFixtures$' -test.v +GOMAXPROCS=1 taskset -c 6 /tmp/producer-pr.test \ + -test.run '^TestProducerBranchComparisonFixtures$' -test.v +``` + +Run each workload/arm in its own process with one iteration. For example: + +```sh +GOMAXPROCS=1 taskset -c 6 /tmp/producer-dev.test \ + -test.run '^$' \ + -test.bench '^BenchmarkProducerBranchHeads$/^maximum-1232B$/^default$' \ + -test.benchmem -test.benchtime=1x -test.count=1 +``` + +Repeat with the PR binary using `default` and `one-fec`, and with `small-215B` +in place of `maximum-1232B`. Alternate dev/default, PR/default, PR/one-fec +and the reverse order over five rounds, as recorded in +[sample-order.json](sample-order.json). Each combined file below contains five +sample outputs in round order, with trailing whitespace trimmed. + +| Workload | dev/default | PR/default | PR/one-fec | +| --- | --- | --- | --- | +| Maximum | [raw](dev-default-maximum-1232B.txt) | [raw](pr-default-maximum-1232B.txt) | [raw](pr-one-fec-maximum-1232B.txt) | +| Small | [raw](dev-default-small-215B.txt) | [raw](pr-default-small-215B.txt) | [raw](pr-one-fec-small-215B.txt) | + +[Method and source digest](method.json), [medians and ranges](summary.json), +[dev validation](dev-validation.txt), [PR validation](pr-validation.txt). diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-default-maximum-1232B.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-default-maximum-1232B.txt new file mode 100644 index 000000000..a5fda678c --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-default-maximum-1232B.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 734051580 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 2129429456 B/op 3844272 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 732608897 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 2129429048 B/op 3844267 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 741401888 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 2129428768 B/op 3844266 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 731935065 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 2129429176 B/op 3844267 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 734241185 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 2129428976 B/op 3844268 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-default-small-215B.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-default-small-215B.txt new file mode 100644 index 000000000..16fdb3596 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-default-small-215B.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 149116346 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 374666112 B/op 1215636 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 151170303 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 374665600 B/op 1215631 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 150767217 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 374665624 B/op 1215632 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 150338635 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 374665272 B/op 1215629 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 149796249 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 374666032 B/op 1215635 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-validation.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-validation.txt new file mode 100644 index 000000000..776a2a325 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/dev-validation.txt @@ -0,0 +1,5 @@ +=== RUN TestProducerBranchComparisonFixtures + producer_branch_bench_test.go:208: wire=215 batch-target=30816 batches=2 packets=512 + producer_branch_bench_test.go:208: wire=1232 batch-target=30816 batches=10 packets=1024 +--- PASS: TestProducerBranchComparisonFixtures (0.07s) +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/method.json b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/method.json new file mode 100644 index 000000000..040c1b6f0 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/method.json @@ -0,0 +1,31 @@ +{ + "dev": { + "sha": "7e4e8af115a27aa15f3401d59f3be1f0d79a4c1b" + }, + "pr": { + "sha": "a598cd0508cab1788c688b8ce667c6c810e1f2d5" + }, + "cpu": "AMD Ryzen 7 9700X", + "go_version": "1.26.4", + "goamd64": "v1", + "cgo_enabled": false, + "gomaxprocs": 1, + "pinned_logical_cpu": 6, + "smt_enabled": true, + "shared_host": true, + "samples_per_arm": 5, + "iterations_per_sample": 1, + "maximum_transaction_slots": [ + 16667, + 16667, + 16666 + ], + "small_transfer_slots": [ + 50000 + ], + "benchmark_source_sha256": "9ca52e8712bc24872693c2b6b43fe2e964bb1292a412601c49570f366d0a09e2", + "production_source_unchanged": true, + "entry_byte_cap_checked": 20971472, + "data_shred_cap_checked": 32768, + "scope": "Serial EntryBuilder and BroadcastSession CPU stage; excludes execution, admission, async worker queue, routing, UDP, receiver." +} diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-default-maximum-1232B.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-default-maximum-1232B.txt new file mode 100644 index 000000000..ecdb1ee11 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-default-maximum-1232B.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 282752023 ns/op 61632 batch-target-B 1023 entry-batches/op 21888 max-data-shreds/slot 20552840 max-entry-B/slot 131328 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 708002984 B/op 974518 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 281298932 ns/op 61632 batch-target-B 1023 entry-batches/op 21888 max-data-shreds/slot 20552840 max-entry-B/slot 131328 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 708002592 B/op 974514 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 282295689 ns/op 61632 batch-target-B 1023 entry-batches/op 21888 max-data-shreds/slot 20552840 max-entry-B/slot 131328 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 708002560 B/op 974514 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 282836822 ns/op 61632 batch-target-B 1023 entry-batches/op 21888 max-data-shreds/slot 20552840 max-entry-B/slot 131328 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 708002592 B/op 974513 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/default 1 282012568 ns/op 61632 batch-target-B 1023 entry-batches/op 21888 max-data-shreds/slot 20552840 max-entry-B/slot 131328 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 708002544 B/op 974513 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-default-small-215B.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-default-small-215B.txt new file mode 100644 index 000000000..6ce4058ec --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-default-small-215B.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 71442020 ns/op 61632 batch-target-B 175.0 entry-batches/op 11296 max-data-shreds/slot 10759800 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141461632 B/op 731204 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 71654018 ns/op 61632 batch-target-B 175.0 entry-batches/op 11296 max-data-shreds/slot 10759800 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141461632 B/op 731204 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 71924395 ns/op 61632 batch-target-B 175.0 entry-batches/op 11296 max-data-shreds/slot 10759800 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141461600 B/op 731204 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 70993771 ns/op 61632 batch-target-B 175.0 entry-batches/op 11296 max-data-shreds/slot 10759800 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141461632 B/op 731204 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/default 1 70602438 ns/op 61632 batch-target-B 175.0 entry-batches/op 11296 max-data-shreds/slot 10759800 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141461944 B/op 731208 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-one-fec-maximum-1232B.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-one-fec-maximum-1232B.txt new file mode 100644 index 000000000..a5b9a27b0 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-one-fec-maximum-1232B.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/one-fec 1 285117925 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 712676592 B/op 997853 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/one-fec 1 288377771 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 712676952 B/op 997857 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/one-fec 1 288468481 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 712676984 B/op 997857 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/one-fec 1 288224324 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 712676328 B/op 997848 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/maximum-1232B/one-fec 1 288659088 ns/op 30816 batch-target-B 2085 entry-batches/op 22336 max-data-shreds/slot 20572664 max-entry-B/slot 134016 packets/op 3.000 slots/op 50000 transactions/op 1232 wire-B/tx 712676760 B/op 997857 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-one-fec-small-215B.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-one-fec-small-215B.txt new file mode 100644 index 000000000..e2a73072b --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-one-fec-small-215B.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/one-fec 1 72424110 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141777944 B/op 735400 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/one-fec 1 72844387 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141777880 B/op 735400 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/one-fec 1 70699409 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141778192 B/op 735404 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/one-fec 1 72504651 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141778192 B/op 735404 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBranchHeads/small-215B/one-fec 1 71786365 ns/op 30816 batch-target-B 350.0 entry-batches/op 11296 max-data-shreds/slot 10769600 max-entry-B/slot 22592 packets/op 1.000 slots/op 50000 transactions/op 215.0 wire-B/tx 141777912 B/op 735400 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-validation.txt b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-validation.txt new file mode 100644 index 000000000..26537e1a9 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/pr-validation.txt @@ -0,0 +1,5 @@ +=== RUN TestProducerBranchComparisonFixtures + producer_branch_bench_test.go:208: wire=215 batch-target=61632 batches=2 packets=512 + producer_branch_bench_test.go:208: wire=1232 batch-target=61632 batches=6 packets=1024 +--- PASS: TestProducerBranchComparisonFixtures (0.06s) +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/sample-order.json b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/sample-order.json new file mode 100644 index 000000000..b4c12e4e1 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/sample-order.json @@ -0,0 +1,242 @@ +[ + { + "round": 1, + "variant": "dev", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "dev-default-small-215B.txt" + }, + { + "round": 1, + "variant": "dev", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "dev-default-maximum-1232B.txt" + }, + { + "round": 1, + "variant": "pr", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-default-small-215B.txt" + }, + { + "round": 1, + "variant": "pr", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-default-maximum-1232B.txt" + }, + { + "round": 1, + "variant": "pr", + "target": "one-fec", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-one-fec-small-215B.txt" + }, + { + "round": 1, + "variant": "pr", + "target": "one-fec", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-one-fec-maximum-1232B.txt" + }, + { + "round": 2, + "variant": "pr", + "target": "one-fec", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-one-fec-small-215B.txt" + }, + { + "round": 2, + "variant": "pr", + "target": "one-fec", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-one-fec-maximum-1232B.txt" + }, + { + "round": 2, + "variant": "pr", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-default-small-215B.txt" + }, + { + "round": 2, + "variant": "pr", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-default-maximum-1232B.txt" + }, + { + "round": 2, + "variant": "dev", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "dev-default-small-215B.txt" + }, + { + "round": 2, + "variant": "dev", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "dev-default-maximum-1232B.txt" + }, + { + "round": 3, + "variant": "dev", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "dev-default-small-215B.txt" + }, + { + "round": 3, + "variant": "dev", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "dev-default-maximum-1232B.txt" + }, + { + "round": 3, + "variant": "pr", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-default-small-215B.txt" + }, + { + "round": 3, + "variant": "pr", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-default-maximum-1232B.txt" + }, + { + "round": 3, + "variant": "pr", + "target": "one-fec", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-one-fec-small-215B.txt" + }, + { + "round": 3, + "variant": "pr", + "target": "one-fec", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-one-fec-maximum-1232B.txt" + }, + { + "round": 4, + "variant": "pr", + "target": "one-fec", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-one-fec-small-215B.txt" + }, + { + "round": 4, + "variant": "pr", + "target": "one-fec", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-one-fec-maximum-1232B.txt" + }, + { + "round": 4, + "variant": "pr", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-default-small-215B.txt" + }, + { + "round": 4, + "variant": "pr", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-default-maximum-1232B.txt" + }, + { + "round": 4, + "variant": "dev", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "dev-default-small-215B.txt" + }, + { + "round": 4, + "variant": "dev", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "dev-default-maximum-1232B.txt" + }, + { + "round": 5, + "variant": "dev", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "dev-default-small-215B.txt" + }, + { + "round": 5, + "variant": "dev", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "dev-default-maximum-1232B.txt" + }, + { + "round": 5, + "variant": "pr", + "target": "default", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-default-small-215B.txt" + }, + { + "round": 5, + "variant": "pr", + "target": "default", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-default-maximum-1232B.txt" + }, + { + "round": 5, + "variant": "pr", + "target": "one-fec", + "workload": "small-215B", + "exit_code": 0, + "combined_file": "pr-one-fec-small-215B.txt" + }, + { + "round": 5, + "variant": "pr", + "target": "one-fec", + "workload": "maximum-1232B", + "exit_code": 0, + "combined_file": "pr-one-fec-maximum-1232B.txt" + } +] diff --git a/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/summary.json b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/summary.json new file mode 100644 index 000000000..fb946f452 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/alpenglow-dev-head/summary.json @@ -0,0 +1,100 @@ +{ + "dev/default/small-215B": { + "ns/op": 150338635.0, + "batch-target-B": 30816.0, + "entry-batches/op": 350.0, + "max-data-shreds/slot": 11296.0, + "max-entry-B/slot": 10769600.0, + "packets/op": 22592.0, + "slots/op": 1.0, + "transactions/op": 50000.0, + "wire-B/tx": 215.0, + "B/op": 374665624.0, + "allocs/op": 1215632.0, + "min_ns": 149116346.0, + "max_ns": 151170303.0 + }, + "dev/default/maximum-1232B": { + "ns/op": 734051580.0, + "batch-target-B": 30816.0, + "entry-batches/op": 2085.0, + "max-data-shreds/slot": 22336.0, + "max-entry-B/slot": 20572664.0, + "packets/op": 134016.0, + "slots/op": 3.0, + "transactions/op": 50000.0, + "wire-B/tx": 1232.0, + "B/op": 2129429048.0, + "allocs/op": 3844267.0, + "min_ns": 731935065.0, + "max_ns": 741401888.0 + }, + "pr/default/small-215B": { + "ns/op": 71442020.0, + "batch-target-B": 61632.0, + "entry-batches/op": 175.0, + "max-data-shreds/slot": 11296.0, + "max-entry-B/slot": 10759800.0, + "packets/op": 22592.0, + "slots/op": 1.0, + "transactions/op": 50000.0, + "wire-B/tx": 215.0, + "B/op": 141461632.0, + "allocs/op": 731204.0, + "min_ns": 70602438.0, + "max_ns": 71924395.0, + "speedup_vs_dev": 2.1043446839829, + "time_reduction_percent": 52.479267887459535 + }, + "pr/default/maximum-1232B": { + "ns/op": 282295689.0, + "batch-target-B": 61632.0, + "entry-batches/op": 1023.0, + "max-data-shreds/slot": 21888.0, + "max-entry-B/slot": 20552840.0, + "packets/op": 131328.0, + "slots/op": 3.0, + "transactions/op": 50000.0, + "wire-B/tx": 1232.0, + "B/op": 708002592.0, + "allocs/op": 974514.0, + "min_ns": 281298932.0, + "max_ns": 282836822.0, + "speedup_vs_dev": 2.6002932690906237, + "time_reduction_percent": 61.54279934933182 + }, + "pr/one-fec/small-215B": { + "ns/op": 72424110.0, + "batch-target-B": 30816.0, + "entry-batches/op": 350.0, + "max-data-shreds/slot": 11296.0, + "max-entry-B/slot": 10769600.0, + "packets/op": 22592.0, + "slots/op": 1.0, + "transactions/op": 50000.0, + "wire-B/tx": 215.0, + "B/op": 141777944.0, + "allocs/op": 735400.0, + "min_ns": 70699409.0, + "max_ns": 72844387.0, + "speedup_vs_dev": 2.0758092160193615, + "time_reduction_percent": 51.8260159805229 + }, + "pr/one-fec/maximum-1232B": { + "ns/op": 288377771.0, + "batch-target-B": 30816.0, + "entry-batches/op": 2085.0, + "max-data-shreds/slot": 22336.0, + "max-entry-B/slot": 20572664.0, + "packets/op": 134016.0, + "slots/op": 3.0, + "transactions/op": 50000.0, + "wire-B/tx": 1232.0, + "B/op": 712676760.0, + "allocs/op": 997857.0, + "min_ns": 285117925.0, + "max_ns": 288659088.0, + "speedup_vs_dev": 2.545451327453391, + "time_reduction_percent": 60.714236048643876 + } +} diff --git a/docs/results/producer-batch/2026-09-06-zen5/before.txt b/docs/results/producer-batch/2026-09-06-zen5/before.txt new file mode 100644 index 000000000..f50a1a3ce --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/before.txt @@ -0,0 +1,40 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 1555476 778.1 ns/op 2142 B/op 14 allocs/op +BenchmarkEntryBuilder/append-and-shred 640376 1887 ns/op 4620 B/op 22 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 618963 1881 ns/op 4621 B/op 22 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 1560634 773.4 ns/op 2142 B/op 14 allocs/op +BenchmarkEntryBuilder/append-and-shred 630884 1888 ns/op 4619 B/op 22 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 637483 1866 ns/op 4620 B/op 22 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 1545517 777.7 ns/op 2142 B/op 14 allocs/op +BenchmarkEntryBuilder/append-and-shred 637521 1878 ns/op 4620 B/op 22 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 640904 1872 ns/op 4620 B/op 22 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 1551872 777.1 ns/op 2142 B/op 14 allocs/op +BenchmarkEntryBuilder/append-and-shred 634971 1887 ns/op 4620 B/op 22 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 608508 1864 ns/op 4620 B/op 22 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkEntryBuilder/append 1541436 783.4 ns/op 2142 B/op 14 allocs/op +BenchmarkEntryBuilder/append-and-shred 628476 1894 ns/op 4620 B/op 22 allocs/op +BenchmarkEntryBuilder/append-and-broadcast 617526 1873 ns/op 4621 B/op 22 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/incremental-accounting.patch b/docs/results/producer-batch/2026-09-06-zen5/incremental-accounting.patch new file mode 100644 index 000000000..16611b22b --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/incremental-accounting.patch @@ -0,0 +1,55 @@ +--- a/pkg/blockprod/entry.go ++++ b/pkg/blockprod/entry.go +@@ -11,0 +12,4 @@ ++// EntryBuilder emits one entry per batch: an entry count, num_hashes, a ++// 32-byte hash, and a transaction count precede the serialized transactions. ++const singleEntryBatchHeaderBytes = 8 + 8 + 32 + 8 ++ +@@ -16,3 +20,4 @@ +- pendingTxns []solana.Transaction +- pendingWire int +- entryHash solana.Hash ++ pendingTxns []solana.Transaction ++ pendingWire int ++ pendingSerializedBytes int ++ entryHash solana.Hash +@@ -38,0 +44,8 @@ ++ // MarshalWithEncoder writes these transaction bytes without framing. Size ++ // each new transaction once rather than reserializing the pending batch. ++ // Keep this separate from the caller's wire-size hint, which may differ ++ // from the serialized representation used in the emitted component. ++ wire, err := tx.MarshalBinary() ++ if err != nil { ++ return nil, 0, false ++ } +@@ -40,4 +52,0 @@ +- wire, err := tx.MarshalBinary() +- if err != nil { +- return nil, 0, false +- } +@@ -47,4 +56 @@ +- nextBytes, err := estimateBatchBytes(b.pendingTxns, tx) +- if err != nil { +- return nil, 0, false +- } ++ nextBytes := singleEntryBatchHeaderBytes + b.pendingSerializedBytes + len(wire) +@@ -54,0 +61 @@ ++ b.pendingSerializedBytes = len(wire) +@@ -59,0 +67 @@ ++ b.pendingSerializedBytes += len(wire) +@@ -89,0 +98 @@ ++ b.pendingSerializedBytes = 0 +@@ -91,13 +99,0 @@ +-} +- +-func estimateBatchBytes(pending []solana.Transaction, next solana.Transaction) (int, error) { +- txns := append(append([]solana.Transaction(nil), pending...), next) +- entries := []turbine.Entry{{ +- NumHashes: 1, +- Txns: txns, +- }} +- out, err := marshalEntryBatchBytes(entries) +- if err != nil { +- return 0, err +- } +- return len(out), nil diff --git a/docs/results/producer-batch/2026-09-06-zen5/max-size/README.md b/docs/results/producer-batch/2026-09-06-zen5/max-size/README.md new file mode 100644 index 000000000..5f22f7a34 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/max-size/README.md @@ -0,0 +1,46 @@ +# Maximum-size transaction workload + +Five samples per version on an AMD Ryzen 7 9700X, Go 1.26.4, Linux amd64, +`GOAMD64=v1`, `CGO_ENABLED=0`, `GOMAXPROCS=1`, pinned to logical CPU 6. +Each sample times one complete 50,000-transaction workload. Version order +alternates original/intermediate/current and current/intermediate/original. +The host was shared, with SMT and frequency boost enabled. + +Each of the 512 precomputed fixtures is a signed legacy system transfer plus +a 981-byte UTF-8 memo, exactly 1,232 bytes in canonical wire form. All fixtures +pass structural sanitization, signature verification, and byte-for-byte +reserialization checks before timing. The fixture pool is reused. + +Each iteration completes two slots of 25,000 transactions, including final +flushes, headers, footers, ending ticks, block IDs, and chained roots. Each +slot produces 32,768 data shreds, fitting the default data-shred budget. +Compute and account budgets are not enforced; transactions are not executed. +A single 50,000-transaction slot at this wire size exceeds the default +32,768 data-shred cap. + +| Producer version | Median ms / 50k transactions | Allocated bytes / workload | Allocations / workload | +| --- | ---: | ---: | ---: | +| Original reviewed implementation | 2,114.572 | 10,132,867,648 | 12,525,832 | +| Incremental size accounting | 384.560 | 1,243,725,728 | 1,663,372 | +| Final implementation | 285.456 | 707,691,696 | 974,131 | + +The combined speedup is **7.41x**, or **86.50% less elapsed time**. The last +two optimizations reduce time by a further 25.77% from the intermediate version. +Dividing the final median by two gives 142.728 ms per 25,000-transaction +slot-generation pass; this is an aggregate average, not measured slot latency. + +All versions produce 1,022 entry batches and 131,072 packets (data plus coding). +A full entry batch holds 49 maximum-size transactions at 60,424 serialized +bytes including its 56-byte header. Total transaction bytes are 61,600,000. + +The timed path includes entry construction, serialization, erasure coding, +Merkle proofs, shred signing, and broadcast-session/block bookkeeping. The +packet sink only counts packets. Fixture signing/parsing/validation happens +before timing. Execution, admission signature verification, routing, UDP, and +receiver processing are excluded. This measures the producer CPU stage, +not whole-validator throughput or the latency of an executed block. + +Source: [BenchmarkProducer50kMaxSize](../../../../../pkg/blockprod/producer_maxsize_bench_test.go). +Raw samples: [original](original.txt), [intermediate](intermediate.txt), +[final](current.txt). Each file contains the five sample outputs (trailing whitespace trimmed) in +sample order. See [baseline definitions and reproduction](../README.md). diff --git a/docs/results/producer-batch/2026-09-06-zen5/max-size/current.txt b/docs/results/producer-batch/2026-09-06-zen5/max-size/current.txt new file mode 100644 index 000000000..45deb01f9 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/max-size/current.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 285456006 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 707691616 B/op 974130 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 285717016 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 707691760 B/op 974131 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 287363139 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 707692088 B/op 974135 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 284528499 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 707691680 B/op 974131 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 284545411 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 707691696 B/op 974131 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/max-size/intermediate.txt b/docs/results/producer-batch/2026-09-06-zen5/max-size/intermediate.txt new file mode 100644 index 000000000..83601bd0e --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/max-size/intermediate.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 385377503 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 1243725528 B/op 1663368 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 378986202 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 1243725608 B/op 1663369 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 390221576 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 1243725856 B/op 1663373 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 384486423 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 1243725872 B/op 1663372 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 384559742 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 1243725728 B/op 1663372 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/max-size/original.txt b/docs/results/producer-batch/2026-09-06-zen5/max-size/original.txt new file mode 100644 index 000000000..f80f00216 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/max-size/original.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 2106644723 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 10132961632 B/op 12526140 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 2114571842 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 10132677856 B/op 12525201 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 2123394398 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 10132771968 B/op 12525508 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 2115793410 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 10132869248 B/op 12525856 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducer50kMaxSize 1 2107312334 ns/op 1022 entry-batches/op 32768 max-data-shreds/slot 131072 packets/op 2.000 slots/op 50000 transactions/op 1232 wire-B/tx 10132867648 B/op 12525832 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/small-transfer/README.md b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/README.md new file mode 100644 index 000000000..16a7d672c --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/README.md @@ -0,0 +1,30 @@ +# Small-transfer transaction workload + +Same hardware and sampling method as the [maximum-size workload](../max-size/README.md): +one pinned logical CPU, `GOMAXPROCS=1`, five alternating one-iteration samples +per version. Each iteration processes 50,000 pre-signed, pre-parsed 215-byte +transfer fixtures through entry batching and the production broadcast session. +The pool contains 512 fixtures and is reused. + +| Producer version | Median ms / 50k transactions | Allocated bytes / workload | Allocations / workload | +| --- | ---: | ---: | ---: | +| Original reviewed implementation | 2,810.088 | 10,374,805,784 | 52,011,173 | +| Incremental size accounting | 97.931 | 232,001,264 | 1,131,417 | +| Final implementation | 75.332 | 141,461,424 | 731,202 | + +The combined speedup is **37.30x**, or **97.32% less elapsed time**. All versions +produce 175 entry batches and 22,592 packets. A full batch holds 286 small +transactions, so the original repeated serialization does substantially more +redundant work than in the maximum-size workload's 49-transaction batches. +The 37x result is specific to this fixture size and baseline. + +The timed path includes final flush, serialization, erasure coding, Merkle +proofs, shred signing, header/footer/ending tick, and block-ID construction. +The packet sink only counts packets. Execution, admission signature +verification, routing, UDP, and receiver processing are excluded. This is +one synthetic slot-generation pass, not a valid executed 50k-transaction block. + +Source: [BenchmarkProducerBlock50k](../../../../../pkg/blockprod/producer_block_bench_test.go). +Raw samples: [original](original.txt), [intermediate](intermediate.txt), +[final](current.txt). Each file contains the five sample outputs (trailing whitespace trimmed) in +sample order. See [baseline definitions and reproduction](../README.md). diff --git a/docs/results/producer-batch/2026-09-06-zen5/small-transfer/current.txt b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/current.txt new file mode 100644 index 000000000..bb71b9596 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/current.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 79850531 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 141461504 B/op 731202 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 75203536 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 141461440 B/op 731202 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 75332047 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 141461360 B/op 731202 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 74934132 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 141461360 B/op 731202 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 77078138 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 141461424 B/op 731202 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/small-transfer/intermediate.txt b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/intermediate.txt new file mode 100644 index 000000000..867485bb7 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/intermediate.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 97930817 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 232001200 B/op 1131417 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 105399577 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 232001576 B/op 1131421 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 103304583 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 232001624 B/op 1131422 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 97647918 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 232001200 B/op 1131417 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 97644912 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 232001264 B/op 1131417 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/small-transfer/original.txt b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/original.txt new file mode 100644 index 000000000..f9447e7f8 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/small-transfer/original.txt @@ -0,0 +1,30 @@ +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 2810087725 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 10374805856 B/op 52011175 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 2809786941 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 10374805920 B/op 52011177 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 2823570810 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 10374805784 B/op 52011173 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 2824579159 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 10374804128 B/op 52011153 allocs/op +PASS +goos: linux +goarch: amd64 +pkg: github.com/Overclock-Validator/mithril/pkg/blockprod +cpu: AMD Ryzen 7 9700X 8-Core Processor +BenchmarkProducerBlock50k 1 2808587024 ns/op 175.0 entry-batches/op 22592 packets/op 50000 transactions/op 215.0 wire-B/tx 10374804152 B/op 52011155 allocs/op +PASS diff --git a/docs/results/producer-batch/2026-09-06-zen5/steady-state.md b/docs/results/producer-batch/2026-09-06-zen5/steady-state.md new file mode 100644 index 000000000..612c02157 --- /dev/null +++ b/docs/results/producer-batch/2026-09-06-zen5/steady-state.md @@ -0,0 +1,55 @@ +# Producer batch serialization and broadcast + +Measured on September 6, 2026 on an AMD Ryzen 7 9700X, Linux amd64, +Go 1.26.4, `GOAMD64=v1`, `CGO_ENABLED=0`, `GOMAXPROCS=1`, pinned to CPU 6. +Each result is the median of five one-second samples. Before/after binaries +ran sequentially, with their order reversed on alternating pairs. The host +was shared, with SMT and frequency boost enabled. + +The baseline is branch `7layer/erasure-repair-performance` at `84776893` +plus the earlier review fixes, including incremental transaction-size +accounting. These measurements isolate two subsequent changes: + +1. Return the already tracked encoded batch size on flush, eliminating the + serialization pass whose bytes were discarded. +2. Retain each FEC root during generation and send generated packets directly + through `BroadcastSession`, eliminating packet parsing, payload copies, + and root reconstruction in the producer path. + +Both versions use the same 61,632-byte batch target and benchmark source. + +| Benchmark | Before ns/tx | After ns/tx | Time reduction | Before B/tx | After B/tx | Before allocs/tx | After allocs/tx | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| `append` | 777.7 | 466.1 | 40.1% | 2,142 | 990 | 14 | 7 | +| `append-and-shred` | 1,887 | 1,545 | 18.1% | 4,620 | 3,468 | 22 | 15 | +| `append-and-broadcast` | 1,872 | 1,407 | 24.8% | 4,620 | 2,817 | 22 | 14 | + +`append-and-broadcast` exercises entry construction, serialization, erasure +coding, Merkle proofs, signing, and broadcast-session root/index bookkeeping. +The packet sink only counts emitted packets. Transactions are signed and +parsed before timing. Transaction execution, admission signature verification, +peer routing, UDP, and receiver processing are excluded. This is a producer +CPU-stage improvement, not a measurement of whole-validator throughput or +transaction latency. + +`append-and-shred` retains the public API that returns owning parsed shred +objects; its improvement mainly reflects the removed flush serialization. +The direct packet path is used by the live broadcast session. + +Run the current benchmark on Linux with: + +```sh +GOMAXPROCS=1 CGO_ENABLED=0 GOAMD64=v1 taskset -c 6 go test ./pkg/blockprod \ + -run '^$' -bench '^BenchmarkEntryBuilder$' -benchmem -benchtime=1s -count=5 +``` + +Raw samples: [before](before.txt), [after](after.txt). + +Validation covers existing golden packet digests; FEC roots matched against +every packet's Merkle proof across unsigned and signed boundary sizes; +packet, index, chained-root, and block-ID equality with the parsed-shred path; +and exact encoded sizes for legacy/v0 transaction batches. A generation +error must leave broadcast packets and commitments unchanged. The affected +repair, turbine, costmodel, and blockprod packages passed race tests, and vet +passed for blockprod, turbine and its subpackages, and the repair simulator CLI. +The packet, root, block-ID, and entry-size checks also passed on the Zen 5. diff --git a/docs/results/repair-sim/RESULTS_TEMPLATE.md b/docs/results/repair-sim/RESULTS_TEMPLATE.md new file mode 100644 index 000000000..530939d1f --- /dev/null +++ b/docs/results/repair-sim/RESULTS_TEMPLATE.md @@ -0,0 +1,22 @@ +# Repair simulation results template + +Record the commit, CPU, Go version, governor/pinning, command, configuration, +and SHA-256 of every raw JSON file. Do not combine logical network time with +measured local CPU time. + +| scenario | availability | slots | FEC/slot | completed | logical slots/s | CPU slots/s | requests | network data shreds | locally recovered shreds | FEC decodes | p50 completion | p95 completion | p99 completion | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| near-tip | near-loss | | | | | | | | | | | | | +| deep-catchup | mixed | | | | | | | | | | | | | +| deep-catchup | sparse | | | | | | | | | | | | | + +Also report: + +- `stage_cpu_ns` by stage; +- `repair_bytes_requested` and `repair_bytes_received`; +- canceled/late, duplicate, lost, and rejected-corrupt responses; +- queue high-water mark; +- spool bytes and complete slots; +- allocations; +- shred-signature cache hits and actual Ed25519 verifications; +- every limitation emitted in the JSON result. diff --git a/pkg/blockprod/bank_test.go b/pkg/blockprod/bank_test.go index fa219dde8..4e793f157 100644 --- a/pkg/blockprod/bank_test.go +++ b/pkg/blockprod/bank_test.go @@ -455,6 +455,19 @@ func TestEntryBuilderFlush(t *testing.T) { assert.Greater(t, batchBytes, 0) } +func TestEntryBuilderDefaultTargetCoalescesTransactions(t *testing.T) { + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{0xcd}) + for seq := uint64(0); seq < 2; seq++ { + wire := txfixture.MustSignedTransferWire(seq) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + entries, _, flushed := builder.Append(*tx, len(wire)) + assert.False(t, flushed) + assert.Empty(t, entries) + } + assert.Equal(t, 2, builder.PendingCount()) +} + func TestControllerWorkingBank(t *testing.T) { controller := NewController() assert.Nil(t, controller.WorkingBank()) diff --git a/pkg/blockprod/entry.go b/pkg/blockprod/entry.go index e9c0c971f..e018af626 100644 --- a/pkg/blockprod/entry.go +++ b/pkg/blockprod/entry.go @@ -1,21 +1,23 @@ package blockprod import ( - "bytes" - "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/turbine" - bin "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" ) +// EntryBuilder emits one entry per batch: an entry count, num_hashes, a +// 32-byte hash, and a transaction count precede the serialized transactions. +const singleEntryBatchHeaderBytes = 8 + 8 + 32 + 8 + // EntryBuilder accumulates forged transactions into Alpenglow-style entry batches. type EntryBuilder struct { limits costmodel.Limits - pendingTxns []solana.Transaction - pendingWire int - entryHash solana.Hash + pendingTxns []solana.Transaction + pendingWire int + pendingSerializedBytes int + entryHash solana.Hash } func NewEntryBuilder(limits costmodel.Limits, entryHash solana.Hash) *EntryBuilder { @@ -35,28 +37,32 @@ func (b *EntryBuilder) PendingWireBytes() int { // Append adds a forged transaction. When the batch byte budget is exceeded it // returns the flushed entry batch and resets the pending buffer. +// Appended transactions must remain immutable; batches retain their nested slices. func (b *EntryBuilder) Append(tx solana.Transaction, wireSize int) ([]turbine.Entry, int, bool) { + // MarshalWithEncoder writes these transaction bytes without framing. Size + // each new transaction once rather than reserializing the pending batch. + // Keep this separate from the caller's wire-size hint, which may differ + // from the serialized representation used in the emitted component. + wire, err := tx.MarshalBinary() + if err != nil { + return nil, 0, false + } if wireSize <= 0 { - wire, err := tx.MarshalBinary() - if err != nil { - return nil, 0, false - } wireSize = len(wire) } - nextBytes, err := estimateBatchBytes(b.pendingTxns, tx) - if err != nil { - return nil, 0, false - } + nextBytes := singleEntryBatchHeaderBytes + b.pendingSerializedBytes + len(wire) if len(b.pendingTxns) > 0 && nextBytes > int(b.limits.MaxBatchBytes) { flushed, batchBytes := b.flushLocked() b.pendingTxns = append(b.pendingTxns[:0], tx) b.pendingWire = wireSize + b.pendingSerializedBytes = len(wire) return flushed, batchBytes, true } b.pendingTxns = append(b.pendingTxns, tx) b.pendingWire += wireSize + b.pendingSerializedBytes += len(wire) return nil, 0, false } @@ -81,49 +87,11 @@ func (b *EntryBuilder) flushLocked() ([]turbine.Entry, int) { Txns: txns, }} b.entryHash = entryHash - batchBytes, err := marshalEntryBatchBytes(entries) - if err != nil { - return nil, 0 - } + // Append already measured each transaction's canonical encoding. The + // entry hash changes the bytes, but not the fixed-size entry header. + batchBytes := singleEntryBatchHeaderBytes + b.pendingSerializedBytes b.pendingTxns = b.pendingTxns[:0] b.pendingWire = 0 - return entries, len(batchBytes) -} - -func estimateBatchBytes(pending []solana.Transaction, next solana.Transaction) (int, error) { - txns := append(append([]solana.Transaction(nil), pending...), next) - entries := []turbine.Entry{{ - NumHashes: 1, - Txns: txns, - }} - out, err := marshalEntryBatchBytes(entries) - if err != nil { - return 0, err - } - return len(out), nil -} - -func marshalEntryBatchBytes(entries []turbine.Entry) ([]byte, error) { - var buf bytes.Buffer - enc := bin.NewEncoderWithEncoding(&buf, bin.EncodingBin) - if err := enc.WriteUint64(uint64(len(entries)), bin.LE); err != nil { - return nil, err - } - for _, entry := range entries { - if err := enc.WriteUint64(entry.NumHashes, bin.LE); err != nil { - return nil, err - } - if err := enc.WriteBytes(entry.Hash[:], false); err != nil { - return nil, err - } - if err := enc.WriteUint64(uint64(len(entry.Txns)), bin.LE); err != nil { - return nil, err - } - for i := range entry.Txns { - if err := entry.Txns[i].MarshalWithEncoder(enc); err != nil { - return nil, err - } - } - } - return buf.Bytes(), nil + b.pendingSerializedBytes = 0 + return entries, batchBytes } diff --git a/pkg/blockprod/entry_bench_test.go b/pkg/blockprod/entry_bench_test.go new file mode 100644 index 000000000..08f9bc5e7 --- /dev/null +++ b/pkg/blockprod/entry_bench_test.go @@ -0,0 +1,75 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" +) + +// Measure the producer's batch accounting both alone and through component +// serialization, shred generation, and broadcast-session bookkeeping. +// Transaction execution, peer routing, and UDP are excluded. +func BenchmarkEntryBuilder(b *testing.B) { + wires := txfixture.PrecomputeTransferPool(512) + txns := make([]solana.Transaction, len(wires)) + for i, wire := range wires { + tx, err := solana.TransactionFromBytes(wire) + if err != nil { + b.Fatal(err) + } + txns[i] = *tx + } + leader := txfixture.PayerPrivateKey() + for _, tc := range []struct { + name string + shred bool + broadcast bool + }{ + {name: "append"}, + {name: "append-and-shred", shred: true}, + {name: "append-and-broadcast", broadcast: true}, + } { + b.Run(tc.name, func(b *testing.B) { + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + shredder := turbine.Shredder{Slot: 100, ParentSlot: 99, Version: 1} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: 100, ParentSlot: 99, Version: 1, + Broadcaster: &benchmarkPacketBroadcaster{}, + }) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + index := i % len(txns) + entries, _, flushed := builder.Append(txns[index], len(wires[index])) + if flushed && tc.broadcast { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + } + if flushed && tc.shred { + component, err := turbine.NewEntryBatch(entries) + if err != nil { + b.Fatal(err) + } + if _, _, _, err := shredder.MakeMerkleShredsFromComponent( + leader, component, false, solana.Hash{}, 0, 0, + ); err != nil { + b.Fatal(err) + } + } + } + }) + } +} + +type benchmarkPacketBroadcaster struct { + packets int +} + +func (b *benchmarkPacketBroadcaster) Broadcast(packets [][]byte) error { + b.packets += len(packets) + return nil +} diff --git a/pkg/blockprod/entry_test.go b/pkg/blockprod/entry_test.go new file mode 100644 index 000000000..1b6821888 --- /dev/null +++ b/pkg/blockprod/entry_test.go @@ -0,0 +1,106 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestEntryBuilderBatchSizingMatchesComponentEncoding(t *testing.T) { + for _, tc := range []struct { + name string + count int + v0 bool + }{ + {name: "legacy", count: 2}, + {name: "mixed-version", count: 2, v0: true}, + {name: "more-than-128-transactions", count: 129, v0: true}, + } { + t.Run(tc.name, func(t *testing.T) { + txns := make([]solana.Transaction, tc.count) + for i := range txns { + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(uint64(i))) + require.NoError(t, err) + if tc.v0 && i%2 != 0 { + tx.Message.SetVersion(solana.MessageVersionV0) + } + txns[i] = *tx + } + component, err := turbine.NewEntryBatch([]turbine.Entry{{NumHashes: 1, Txns: txns}}) + require.NoError(t, err) + encoded, err := turbine.MarshalBlockComponent(component) + require.NoError(t, err) + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = uint64(len(encoded)) + builder := NewEntryBuilder(limits, solana.Hash{1}) + + checkBatch := func(entries []turbine.Entry, batchBytes int, want []solana.Transaction) { + t.Helper() + require.Len(t, entries, 1) + require.Equal(t, want, entries[0].Txns) + component, err := turbine.NewEntryBatch(entries) + require.NoError(t, err) + encoded, err := turbine.MarshalBlockComponent(component) + require.NoError(t, err) + require.Equal(t, len(encoded), batchBytes) + } + + // Repeat after both an automatic and explicit flush to catch stale + // size accounting. The limit fits the complete batch exactly. + for round := 0; round < 2; round++ { + pendingWire := 0 + for i := range txns { + wire, err := txns[i].MarshalBinary() + require.NoError(t, err) + wireHint := len(wire) + switch i % 3 { + case 0: + wireHint += 100 // Transport bytes must not affect batch sizing. + case 1: + wireHint = 0 // An absent hint must use the serialized length. + } + if wireHint > 0 { + pendingWire += wireHint + } else { + pendingWire += len(wire) + } + entries, _, flushed := builder.Append(txns[i], wireHint) + require.False(t, flushed, "transaction %d", i) + require.Empty(t, entries) + } + require.Equal(t, tc.count, builder.PendingCount()) + require.Equal(t, pendingWire, builder.PendingWireBytes()) + entries, batchBytes, flushed := builder.Append(txns[0], 0) + require.True(t, flushed) + checkBatch(entries, batchBytes, txns) + require.Equal(t, int(limits.MaxBatchBytes), batchBytes) + require.Equal(t, 1, builder.PendingCount()) + entries, batchBytes = builder.Flush() + checkBatch(entries, batchBytes, txns[:1]) + require.Zero(t, builder.PendingCount()) + require.Zero(t, builder.PendingWireBytes()) + } + }) + } +} + +func TestEntryBuilderAllowsSingleTransactionOverBatchTarget(t *testing.T) { + wire := txfixture.MustSignedTransferWire(0) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = 1 + builder := NewEntryBuilder(limits, solana.Hash{}) + entries, _, flushed := builder.Append(*tx, len(wire)) + require.False(t, flushed) + require.Empty(t, entries) + entries, _, flushed = builder.Append(*tx, len(wire)) + require.True(t, flushed) + require.Len(t, entries, 1) + require.Len(t, entries[0].Txns, 1) + require.Equal(t, 1, builder.PendingCount()) +} diff --git a/pkg/blockprod/producer_block_bench_test.go b/pkg/blockprod/producer_block_bench_test.go new file mode 100644 index 000000000..c93c6849d --- /dev/null +++ b/pkg/blockprod/producer_block_bench_test.go @@ -0,0 +1,66 @@ +package blockprod + +import ( + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "testing" +) + +func BenchmarkProducerBlock50k(b *testing.B) { + const transactions = 50000 + wires := txfixture.PrecomputeTransferPool(512) + txns := make([]solana.Transaction, len(wires)) + for i, wire := range wires { + tx, err := solana.TransactionFromBytes(wire) + if err != nil { + b.Fatal(err) + } + txns[i] = *tx + } + leader := txfixture.PayerPrivateKey() + var lastBatches, lastPackets int + b.ReportAllocs() + b.ResetTimer() + for round := 0; round < b.N; round++ { + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + sink := &benchmarkPacketBroadcaster{} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: 100, ParentSlot: 99, Version: 1, Broadcaster: sink, + }) + if err := session.BroadcastHeader(solana.Hash{}); err != nil { + b.Fatal(err) + } + batches := 0 + for i := 0; i < transactions; i++ { + idx := i % len(txns) + entries, _, flushed := builder.Append(txns[idx], len(wires[idx])) + if flushed { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + batches++ + } + } + if entries, _ := builder.Flush(); len(entries) > 0 { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + batches++ + } + if err := session.BroadcastFooter(solana.Hash{1}, 0, nil, nil); err != nil { + b.Fatal(err) + } + if err := session.BroadcastEndingTickLast(builder.CurrentEntryHash()); err != nil { + b.Fatal(err) + } + _ = session.BlockID(99, solana.Hash{}) + lastBatches, lastPackets = batches, sink.packets + } + b.StopTimer() + b.ReportMetric(transactions, "transactions/op") + b.ReportMetric(float64(len(wires[0])), "wire-B/tx") + b.ReportMetric(float64(lastBatches), "entry-batches/op") + b.ReportMetric(float64(lastPackets), "packets/op") +} diff --git a/pkg/blockprod/producer_branch_bench_test.go b/pkg/blockprod/producer_branch_bench_test.go new file mode 100644 index 000000000..2d04fd1c4 --- /dev/null +++ b/pkg/blockprod/producer_branch_bench_test.go @@ -0,0 +1,253 @@ +package blockprod + +import ( + "bytes" + "fmt" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + txwire "github.com/Overclock-Validator/mithril/pkg/tpu/wire" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" +) + +// These complete producer CPU workloads compare branch heads using identical +// pre-signed, parsed fixtures and a packet-counting sink. No execution, admission, +// worker queue, routing, UDP, or receiver work is timed. +const branchComparisonOneFECBytes = 32 * 963 +const branchComparisonSlotEntryCap = 20*1024*1024 - 48 + +func branchMaxSizeTransferPool(tb testing.TB) ([][]byte, []solana.Transaction) { + tb.Helper() + payer, destination := txfixture.PayerPubkey(), txfixture.DestPubkey() + privateKey := txfixture.PayerPrivateKey() + wires := make([][]byte, 512) + txns := make([]solana.Transaction, len(wires)) + for i := range wires { + memo := bytes.Repeat([]byte("m"), 981) + copy(memo, fmt.Sprintf("mithril-max-wire-%04d:", i)) + tx, err := solana.NewTransaction([]solana.Instruction{ + system.NewTransferInstruction(uint64(i+1), payer, destination).Build(), + solana.NewInstruction(solana.MemoProgramID, nil, memo), + }, txfixture.TestBlockhash(), solana.TransactionPayer(payer)) + if err != nil { + tb.Fatal(err) + } + _, err = tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key == payer { + return &privateKey + } + return nil + }) + if err != nil { + tb.Fatal(err) + } + raw, err := tx.MarshalBinary() + if err != nil { + tb.Fatal(err) + } + if len(raw) != txwire.PacketDataSize { + tb.Fatalf("wire size %d, expected %d", len(raw), txwire.PacketDataSize) + } + if _, err := txwire.Sanitize(raw); err != nil { + tb.Fatal(err) + } + decoded, err := solana.TransactionFromBytes(raw) + if err != nil { + tb.Fatal(err) + } + if err := decoded.VerifySignatures(); err != nil { + tb.Fatal(err) + } + encoded, err := decoded.MarshalBinary() + if err != nil || !bytes.Equal(encoded, raw) { + tb.Fatal("canonical transaction round trip changed bytes") + } + wires[i], txns[i] = raw, *decoded + } + return wires, txns +} + +func branchComparisonFixtures(tb testing.TB, maximum bool) ([][]byte, []solana.Transaction) { + tb.Helper() + if maximum { + return branchMaxSizeTransferPool(tb) + } + wires := txfixture.PrecomputeTransferPool(512) + txns := make([]solana.Transaction, len(wires)) + for i, raw := range wires { + if len(raw) != 215 { + tb.Fatalf("transfer size %d, want 215", len(raw)) + } + if _, err := txwire.Sanitize(raw); err != nil { + tb.Fatal(err) + } + tx, err := solana.TransactionFromBytes(raw) + if err != nil { + tb.Fatal(err) + } + if err := tx.VerifySignatures(); err != nil { + tb.Fatal(err) + } + encoded, err := tx.MarshalBinary() + if err != nil || !bytes.Equal(encoded, raw) { + tb.Fatal("transfer round trip changed bytes") + } + txns[i] = *tx + } + return wires, txns +} + +type branchComparisonSink struct{ packets int } + +func (s *branchComparisonSink) Broadcast(packets [][]byte) error { + s.packets += len(packets) + return nil +} + +type branchComparisonStats struct { + batches, packets, transactions, maxDataShreds, maxEntryBytes int + blockID solana.Hash +} + +func branchComparisonRun(tb testing.TB, wires [][]byte, txns []solana.Transaction, slots []int, limits costmodel.Limits) branchComparisonStats { + tb.Helper() + var stats branchComparisonStats + var parentID, parentRoot solana.Hash + leader := txfixture.PayerPrivateKey() + for number, count := range slots { + slot := uint64(100 + number) + builder := NewEntryBuilder(limits, solana.Hash{}) + sink := &branchComparisonSink{} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: slot, ParentSlot: slot - 1, Version: 1, Broadcaster: sink, + ParentBlockID: parentID, ParentChainedMerkleRoot: parentRoot, + }) + if err := session.BroadcastHeader(parentID); err != nil { + tb.Fatal(err) + } + entryBytes := 0 + for i := 0; i < count; i++ { + index := stats.transactions % len(txns) + entries, size, flushed := builder.Append(txns[index], len(wires[index])) + stats.transactions++ + if flushed { + if err := session.BroadcastEntryBatch(entries); err != nil { + tb.Fatal(err) + } + stats.batches++ + entryBytes += size + } + } + if entries, size := builder.Flush(); len(entries) != 0 { + if err := session.BroadcastEntryBatch(entries); err != nil { + tb.Fatal(err) + } + stats.batches++ + entryBytes += size + } + if err := session.BroadcastFooter(solana.Hash{1}, 0, nil, nil); err != nil { + tb.Fatal(err) + } + if err := session.BroadcastEndingTickLast(builder.CurrentEntryHash()); err != nil { + tb.Fatal(err) + } + parentID = session.BlockID(slot-1, parentID) + parentRoot = session.ChainedMerkleRoot() + dataShreds := sink.packets / 2 + if dataShreds > costmodel.DefaultMaxDataShredsPerSlot { + tb.Fatal("slot exceeds data-shred budget") + } + if entryBytes > branchComparisonSlotEntryCap { + tb.Fatal("slot exceeds entry-byte budget") + } + if dataShreds > stats.maxDataShreds { + stats.maxDataShreds = dataShreds + } + if entryBytes > stats.maxEntryBytes { + stats.maxEntryBytes = entryBytes + } + stats.packets += sink.packets + } + stats.blockID = parentID + return stats +} + +func branchComparisonCheck(tb testing.TB, got branchComparisonStats, wireSize int, slots []int, batchLimit uint64) { + tb.Helper() + perBatch := (int(batchLimit) - 56) / wireSize + var batches, packets, transactions int + for _, count := range slots { + full, tail := count/perBatch, count%perBatch + batches += full + fecs := full * ((56 + perBatch*wireSize + branchComparisonOneFECBytes - 1) / branchComparisonOneFECBytes) + if tail != 0 { + batches++ + fecs += (56 + tail*wireSize + branchComparisonOneFECBytes - 1) / branchComparisonOneFECBytes + } + packets += (fecs + 3) * 64 // Header, footer, and signed ending tick each use one FEC set. + transactions += count + } + if got.batches != batches || got.packets != packets || got.transactions != transactions { + tb.Fatalf("workload counts %+v, want batches=%d packets=%d transactions=%d", got, batches, packets, transactions) + } + if got.blockID == (solana.Hash{}) { + tb.Fatal("empty block commitment") + } +} + +func TestProducerBranchComparisonFixtures(t *testing.T) { + for _, maximum := range []bool{false, true} { + wires, txns := branchComparisonFixtures(t, maximum) + limits := costmodel.DefaultLimits() + slots := []int{101, 100} + got := branchComparisonRun(t, wires, txns, slots, limits) + branchComparisonCheck(t, got, len(wires[0]), slots, limits.MaxBatchBytes) + t.Logf("wire=%d batch-target=%d batches=%d packets=%d", len(wires[0]), limits.MaxBatchBytes, got.batches, got.packets) + } +} + +func BenchmarkProducerBranchHeads(b *testing.B) { + for _, workload := range []struct { + name string + maximum bool + slots []int + }{ + {name: "small-215B", slots: []int{50000}}, + {name: "maximum-1232B", maximum: true, slots: []int{16667, 16667, 16666}}, + } { + wires, txns := branchComparisonFixtures(b, workload.maximum) + for _, target := range []struct { + name string + bytes uint64 + }{ + {name: "default"}, + {name: "one-fec", bytes: branchComparisonOneFECBytes}, + } { + b.Run(workload.name+"/"+target.name, func(b *testing.B) { + limits := costmodel.DefaultLimits() + if target.bytes != 0 { + limits.MaxBatchBytes = target.bytes + } + var last branchComparisonStats + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + last = branchComparisonRun(b, wires, txns, workload.slots, limits) + } + b.StopTimer() + branchComparisonCheck(b, last, len(wires[0]), workload.slots, limits.MaxBatchBytes) + b.ReportMetric(float64(last.transactions), "transactions/op") + b.ReportMetric(float64(len(workload.slots)), "slots/op") + b.ReportMetric(float64(len(wires[0])), "wire-B/tx") + b.ReportMetric(float64(limits.MaxBatchBytes), "batch-target-B") + b.ReportMetric(float64(last.batches), "entry-batches/op") + b.ReportMetric(float64(last.packets), "packets/op") + b.ReportMetric(float64(last.maxDataShreds), "max-data-shreds/slot") + b.ReportMetric(float64(last.maxEntryBytes), "max-entry-B/slot") + }) + } + } +} diff --git a/pkg/blockprod/producer_maxsize_bench_test.go b/pkg/blockprod/producer_maxsize_bench_test.go new file mode 100644 index 000000000..966c51850 --- /dev/null +++ b/pkg/blockprod/producer_maxsize_bench_test.go @@ -0,0 +1,153 @@ +package blockprod + +import ( + "bytes" + "fmt" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + txwire "github.com/Overclock-Validator/mithril/pkg/tpu/wire" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" +) + +// Generate exactly 1,232-byte, single-signature legacy transactions. Padding is +// a real UTF-8 memo instruction, rather than trailing bytes after a transaction. +func maxSizeTransferPool(tb testing.TB) ([][]byte, []solana.Transaction) { + tb.Helper() + payer, destination := txfixture.PayerPubkey(), txfixture.DestPubkey() + privateKey := txfixture.PayerPrivateKey() + wires := make([][]byte, 512) + txns := make([]solana.Transaction, len(wires)) + for i := range wires { + memo := bytes.Repeat([]byte("m"), 981) + copy(memo, fmt.Sprintf("mithril-max-wire-%04d:", i)) + tx, err := solana.NewTransaction([]solana.Instruction{ + system.NewTransferInstruction(uint64(i+1), payer, destination).Build(), + solana.NewInstruction(solana.MemoProgramID, nil, memo), + }, txfixture.TestBlockhash(), solana.TransactionPayer(payer)) + if err != nil { + tb.Fatal(err) + } + _, err = tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key == payer { + return &privateKey + } + return nil + }) + if err != nil { + tb.Fatal(err) + } + raw, err := tx.MarshalBinary() + if err != nil { + tb.Fatal(err) + } + if len(raw) != txwire.PacketDataSize { + tb.Fatalf("wire size %d, expected %d", len(raw), txwire.PacketDataSize) + } + if _, err := txwire.Sanitize(raw); err != nil { + tb.Fatal(err) + } + decoded, err := solana.TransactionFromBytes(raw) + if err != nil { + tb.Fatal(err) + } + if err := decoded.VerifySignatures(); err != nil { + tb.Fatal(err) + } + encoded, err := decoded.MarshalBinary() + if err != nil || !bytes.Equal(encoded, raw) { + tb.Fatal("canonical transaction round trip changed bytes") + } + wires[i], txns[i] = raw, *decoded + } + return wires, txns +} + +func TestMaxSizeProducerFixture(t *testing.T) { + wires, txns := maxSizeTransferPool(t) + t.Logf("verified %d signed transactions at %d bytes each", len(wires), len(wires[0])) + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + for i := 0; i < 50; i++ { + entries, batchBytes, flushed := builder.Append(txns[i], len(wires[i])) + if i < 49 && flushed { + t.Fatalf("premature flush at %d", i) + } + if i == 49 { + if !flushed || len(entries) != 1 || len(entries[0].Txns) != 49 || batchBytes != 60424 { + t.Fatal("unexpected full-batch size") + } + } + } +} + +// 50k maximum-size transactions require two slots under the current 32,768 +// data-shred cap. Each iteration completes two slots of 25k transactions. +func BenchmarkProducer50kMaxSize(b *testing.B) { + const transactions = 50000 + const transactionsPerSlot = 25000 + wires, txns := maxSizeTransferPool(b) + leader := txfixture.PayerPrivateKey() + var lastBatches, lastPackets, lastMaxSlotDataShreds int + b.ReportAllocs() + b.ResetTimer() + for round := 0; round < b.N; round++ { + var parentID, parentRoot solana.Hash + totalBatches, totalPackets, maxSlotDataShreds := 0, 0, 0 + for block := 0; block < transactions/transactionsPerSlot; block++ { + slot := uint64(100 + block) + builder := NewEntryBuilder(costmodel.DefaultLimits(), solana.Hash{}) + sink := &benchmarkPacketBroadcaster{} + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: leader, Slot: slot, ParentSlot: slot - 1, Version: 1, Broadcaster: sink, + ParentBlockID: parentID, ParentChainedMerkleRoot: parentRoot, + }) + if err := session.BroadcastHeader(parentID); err != nil { + b.Fatal(err) + } + for i := 0; i < transactionsPerSlot; i++ { + idx := (block*transactionsPerSlot + i) % len(txns) + entries, _, flushed := builder.Append(txns[idx], len(wires[idx])) + if flushed { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + totalBatches++ + } + } + if entries, _ := builder.Flush(); len(entries) > 0 { + if err := session.BroadcastEntryBatch(entries); err != nil { + b.Fatal(err) + } + totalBatches++ + } + if err := session.BroadcastFooter(solana.Hash{1}, 0, nil, nil); err != nil { + b.Fatal(err) + } + if err := session.BroadcastEndingTickLast(builder.CurrentEntryHash()); err != nil { + b.Fatal(err) + } + parentID = session.BlockID(slot-1, parentID) + parentRoot = session.ChainedMerkleRoot() + // The generator emits equal numbers of data and coding shreds. + dataShreds := sink.packets / 2 + if dataShreds > costmodel.DefaultMaxDataShredsPerSlot { + b.Fatal("slot exceeds data-shred limit") + } + if dataShreds > maxSlotDataShreds { + maxSlotDataShreds = dataShreds + } + totalPackets += sink.packets + } + lastBatches, lastPackets, lastMaxSlotDataShreds = totalBatches, totalPackets, maxSlotDataShreds + } + b.StopTimer() + b.ReportMetric(transactions, "transactions/op") + b.ReportMetric(transactions/transactionsPerSlot, "slots/op") + b.ReportMetric(float64(len(wires[0])), "wire-B/tx") + b.ReportMetric(float64(lastBatches), "entry-batches/op") + b.ReportMetric(float64(lastPackets), "packets/op") + b.ReportMetric(float64(lastMaxSlotDataShreds), "max-data-shreds/slot") +} diff --git a/pkg/costmodel/limits.go b/pkg/costmodel/limits.go index fd73c275c..bdbe4190f 100644 --- a/pkg/costmodel/limits.go +++ b/pkg/costmodel/limits.go @@ -4,12 +4,12 @@ package costmodel const ( ComputeUnitToUSRatio = 30 - SignatureCost = ComputeUnitToUSRatio * 24 // 720 + SignatureCost = ComputeUnitToUSRatio * 24 // 720 Secp256k1VerifyCost = ComputeUnitToUSRatio * 223 Ed25519VerifyStrictCost = ComputeUnitToUSRatio * 80 Secp256r1VerifyCost = ComputeUnitToUSRatio * 160 - WriteLockUnits = ComputeUnitToUSRatio * 10 // 300 - InstructionDataBytesCost = 140 / ComputeUnitToUSRatio // ~4 CU per byte + WriteLockUnits = ComputeUnitToUSRatio * 10 // 300 + InstructionDataBytesCost = 140 / ComputeUnitToUSRatio // ~4 CU per byte MaxBlockUnitsSIMD0256 = 60_000_000 MaxBlockUnitsSIMD0286 = 100_000_000 @@ -20,9 +20,15 @@ const ( // DefaultMaxDataShredsPerSlot matches agave DEFAULT_MAX_DATA_SHREDS_PER_SLOT. DefaultMaxDataShredsPerSlot = 32 * 1024 - // TypicalDataShredPayloadBytes is the usable data bytes per entry-batch target. + // TypicalDataShredPayloadBytes is the usable data in one chained Merkle + // data shred for the standard 32+32 FEC layout. TypicalDataShredPayloadBytes = 963 - DefaultTargetBatchBytes = 2 * TypicalDataShredPayloadBytes + // DataShredsPerFECBlock and DefaultTargetBatchBytes mirror Agave's + // DATA_SHREDS_PER_FEC_BLOCK and get_target_batch_bytes_default. The target + // is two complete FEC payloads, not two individual shred payloads. + DataShredsPerFECBlock = 32 + TypicalFECDataBytes = DataShredsPerFECBlock * TypicalDataShredPayloadBytes + DefaultTargetBatchBytes = 2 * TypicalFECDataBytes ) // Limits configures per-slot cost and size budgets. diff --git a/pkg/costmodel/limits_test.go b/pkg/costmodel/limits_test.go new file mode 100644 index 000000000..e9d32ab2d --- /dev/null +++ b/pkg/costmodel/limits_test.go @@ -0,0 +1,13 @@ +package costmodel + +import "testing" + +func TestDefaultTargetBatchBytesMatchesTwoTypicalFECSets(t *testing.T) { + const want = 61_632 + if DefaultTargetBatchBytes != want { + t.Fatalf("DefaultTargetBatchBytes = %d, want %d", DefaultTargetBatchBytes, want) + } + if DefaultTargetBatchBytes != 2*DataShredsPerFECBlock*TypicalDataShredPayloadBytes { + t.Fatal("default batch target must remain two complete typical FEC payloads") + } +} diff --git a/pkg/turbine/assembler.go b/pkg/turbine/assembler.go index 9b69f4370..5824f13c2 100644 --- a/pkg/turbine/assembler.go +++ b/pkg/turbine/assembler.go @@ -10,6 +10,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/block" "github.com/Overclock-Validator/mithril/pkg/statsd" + "github.com/Overclock-Validator/mithril/pkg/turbine/internal/rsrecover" "github.com/gagliardetto/solana-go" "github.com/klauspost/reedsolomon" ) @@ -1300,21 +1301,43 @@ func (a *SlotAssembler) recoverFEC(state *slotState, fecSetIndex uint32) ([]*Shr } shards[int(layout.dataShreds)+int(pos)] = shard } - encoder, err := a.fecEncoder(layout) - if err != nil { - return nil, err - } - required := make([]bool, int(layout.dataShreds)+int(layout.codingShreds)) var missingData int + missingDataIndex := -1 for idx := 0; idx < int(layout.dataShreds); idx++ { if fec.data[uint32(idx)] == nil { - required[idx] = true missingData++ + missingDataIndex = idx } } if missingData == 0 { return nil, nil } + if missingData == 1 && + layout.dataShreds == rsrecover.DataShards && + layout.codingShreds == rsrecover.CodingShards { + presence, err := rsrecover.Presence(shards) + if err != nil { + return nil, err + } + dst := make([]byte, layout.shardSize) + if err := rsrecover.RecoverOneData(presence, missingDataIndex, shards, dst); err != nil { + return nil, fmt.Errorf("recover one FEC data shred slot %d fec_set=%d: %w", state.slot, fecSetIndex, err) + } + shred, err := fec.recoveredDataShred(uint32(missingDataIndex), dst) + if err != nil { + return nil, err + } + return []*Shred{shred}, nil + } + + required := make([]bool, int(layout.dataShreds)+int(layout.codingShreds)) + for idx := 0; idx < int(layout.dataShreds); idx++ { + required[idx] = fec.data[uint32(idx)] == nil + } + encoder, err := a.fecEncoder(layout) + if err != nil { + return nil, err + } if err := encoder.ReconstructSome(shards, required); err != nil { if errors.Is(err, reedsolomon.ErrTooFewShards) { return nil, nil diff --git a/pkg/turbine/broadcast.go b/pkg/turbine/broadcast.go index fee229158..814951132 100644 --- a/pkg/turbine/broadcast.go +++ b/pkg/turbine/broadcast.go @@ -3,7 +3,6 @@ package turbine import ( "fmt" "net" - "sort" "sync" "github.com/gagliardetto/solana-go" @@ -116,8 +115,8 @@ type BroadcastSessionConfig struct { // It seeds the chained merkle root embedded in this slot's first FEC batch. ParentChainedMerkleRoot solana.Hash Broadcaster PacketBroadcaster - UserAgent []byte - Version uint16 + UserAgent []byte + Version uint16 } func NewBroadcastSession(cfg BroadcastSessionConfig) *BroadcastSession { @@ -192,7 +191,7 @@ func (s *BroadcastSession) broadcastComponent(component BlockComponent, isLastIn if s.broadcaster == nil { return nil } - batch, nextData, nextCode, err := s.shredder.MakeMerkleShredsFromComponent( + batch, nextData, nextCode, err := s.shredder.makeMerklePacketsFromComponent( s.leader, component, isLastInSlot, @@ -203,44 +202,9 @@ func (s *BroadcastSession) broadcastComponent(component BlockComponent, isLastIn if err != nil { return err } - s.chainedMerkleRoot = batch.ChainedMerkleRoot + s.chainedMerkleRoot = batch.chainedMerkleRoot s.nextDataIndex = nextData s.nextCodeIndex = nextCode - if len(batch.DataShreds) > 0 { - s.fecSetRoots = appendFECSetMerkleRoots(s.fecSetRoots, batch.DataShreds) - } - return s.broadcaster.Broadcast(batch.Packets) -} - -func appendFECSetMerkleRoots(roots []solana.Hash, dataShreds []*Shred) []solana.Hash { - if len(dataShreds) == 0 { - return roots - } - indices := make([]uint32, 0) - seen := make(map[uint32]struct{}) - for _, shred := range dataShreds { - if shred == nil { - continue - } - if _, ok := seen[shred.FECSetIndex]; ok { - continue - } - seen[shred.FECSetIndex] = struct{}{} - indices = append(indices, shred.FECSetIndex) - } - sort.Slice(indices, func(i, j int) bool { return indices[i] < indices[j] }) - for _, fecSetIndex := range indices { - for _, shred := range dataShreds { - if shred == nil || shred.FECSetIndex != fecSetIndex { - continue - } - root, err := shred.MerkleRoot() - if err != nil { - continue - } - roots = append(roots, root) - break - } - } - return roots + s.fecSetRoots = append(s.fecSetRoots, batch.fecSetRoots...) + return s.broadcaster.Broadcast(batch.packets) } diff --git a/pkg/turbine/broadcast_test.go b/pkg/turbine/broadcast_test.go index 09e82a397..11c211c76 100644 --- a/pkg/turbine/broadcast_test.go +++ b/pkg/turbine/broadcast_test.go @@ -52,6 +52,70 @@ func TestBroadcastSessionHeaderAndFooter(t *testing.T) { require.NotEqual(t, parentBlockID, chainedRoot) } +func TestBroadcastSessionMatchesParsedShreds(t *testing.T) { + leader := testBroadcastLeader(t) + parentID, parentRoot := solana.Hash{0xaa}, solana.Hash{0xbb} + capture := &packetCapture{} + session := NewBroadcastSession(BroadcastSessionConfig{ + Leader: leader, Slot: 100, ParentSlot: 99, Version: 7, + ParentChainedMerkleRoot: parentRoot, Broadcaster: capture, + }) + shredder := Shredder{Slot: 100, ParentSlot: 99, Version: 7} + txns := make([]solana.Transaction, 400) + for i := range txns { + txns[i] = mustParseTransferTx(t, uint64(i)) + } + entries, err := NewEntryBatch([]Entry{{NumHashes: 1, Hash: solana.Hash{1}, Txns: txns}}) + require.NoError(t, err) + tick, err := NewEntryBatch([]Entry{{NumHashes: 1, Hash: solana.Hash{2}}}) + require.NoError(t, err) + components := []BlockComponent{ + NewBlockHeader(99, parentID), + entries, // More than two FEC sets: every root must enter the block ID. + NewUpdateParent(98, solana.Hash{3}), + NewBlockFooter(BlockFooter{BankHash: solana.Hash{4}}), + tick, + } + var nextData, nextCode uint32 + root := parentRoot + var roots []solana.Hash + for i, component := range components { + last := i == len(components)-1 + batch, data, code, err := shredder.MakeMerkleShredsFromComponent( + leader, component, last, root, nextData, nextCode, + ) + require.NoError(t, err) + if i == 1 { + require.Greater(t, len(batch.DataShreds), 2*dataShredsPerFECBlock) + } + for j, shred := range batch.DataShreds { + if j == 0 || shred.FECSetIndex != batch.DataShreds[j-1].FECSetIndex { + fecRoot, err := shred.MerkleRoot() + require.NoError(t, err) + roots = append(roots, fecRoot) + } + } + capture.packets = nil + require.NoError(t, session.BroadcastComponent(component, last)) + require.Equal(t, batch.Packets, capture.packets) + require.Equal(t, batch.ChainedMerkleRoot, session.ChainedMerkleRoot()) + require.Equal(t, data, session.nextDataIndex) + require.Equal(t, code, session.nextCodeIndex) + require.Equal(t, roots, session.fecSetRoots) + require.Equal(t, DoubleMerkleBlockID(99, parentID, roots), session.BlockID(99, parentID)) + root, nextData, nextCode = batch.ChainedMerkleRoot, data, code + } + + // Invalid components must not publish packets or advance the commitment. + capture.packets = nil + require.Error(t, session.BroadcastComponent(BlockComponent{Marker: &BlockMarker{Kind: 255}}, false)) + require.Empty(t, capture.packets) + require.Equal(t, root, session.ChainedMerkleRoot()) + require.Equal(t, nextData, session.nextDataIndex) + require.Equal(t, nextCode, session.nextCodeIndex) + require.Equal(t, roots, session.fecSetRoots) +} + func TestUDPBroadcasterLoopback(t *testing.T) { recvAddr, err := net.ResolveUDPAddr("udp", "127.0.0.1:0") require.NoError(t, err) diff --git a/pkg/turbine/component_shredder.go b/pkg/turbine/component_shredder.go index ad4c9229b..109408a4c 100644 --- a/pkg/turbine/component_shredder.go +++ b/pkg/turbine/component_shredder.go @@ -14,7 +14,7 @@ type Shredder struct { ReferenceTick uint8 } -// ShredBatch is one FEC batch emitted for a single block component. +// ShredBatch contains the FEC batches emitted for a single block component. type ShredBatch struct { Slot uint64 Component BlockComponent @@ -34,23 +34,8 @@ func (s *Shredder) MakeMerkleShredsFromComponent( nextShredIndex uint32, nextCodeIndex uint32, ) (ShredBatch, uint32, uint32, error) { - bytes, err := MarshalBlockComponent(component) - if err != nil { - return ShredBatch{}, nextShredIndex, nextCodeIndex, err - } - gen := ShredGenerator{ - Slot: s.Slot, - ParentSlot: s.ParentSlot, - Version: s.Version, - ReferenceTick: s.ReferenceTick, - } - packets, root, nextData, nextCode, err := gen.MakeShredsFromData( - leader, - bytes, - isLastInSlot, - chainedMerkleRoot, - nextShredIndex, - nextCodeIndex, + generated, nextData, nextCode, err := s.makeMerklePacketsFromComponent( + leader, component, isLastInSlot, chainedMerkleRoot, nextShredIndex, nextCodeIndex, ) if err != nil { return ShredBatch{}, nextShredIndex, nextCodeIndex, err @@ -58,11 +43,11 @@ func (s *Shredder) MakeMerkleShredsFromComponent( batch := ShredBatch{ Slot: s.Slot, Component: component, - Packets: packets, - ChainedMerkleRoot: root, + Packets: generated.packets, + ChainedMerkleRoot: generated.chainedMerkleRoot, IsLastInSlot: isLastInSlot, } - for _, packet := range packets { + for _, packet := range batch.Packets { shred, err := ParseShred(packet) if err != nil { return ShredBatch{}, nextShredIndex, nextCodeIndex, fmt.Errorf("parse generated shred: %w", err) @@ -75,3 +60,37 @@ func (s *Shredder) MakeMerkleShredsFromComponent( } return batch, nextData, nextCode, nil } + +// makeMerklePacketsFromComponent serves the producer, which needs the wire +// packets and FEC roots but not the owning Shred objects exposed by the public API. +func (s *Shredder) makeMerklePacketsFromComponent( + leader solana.PrivateKey, + component BlockComponent, + isLastInSlot bool, + chainedMerkleRoot solana.Hash, + nextShredIndex uint32, + nextCodeIndex uint32, +) (shredPackets, uint32, uint32, error) { + bytes, err := MarshalBlockComponent(component) + if err != nil { + return shredPackets{}, nextShredIndex, nextCodeIndex, err + } + gen := ShredGenerator{ + Slot: s.Slot, + ParentSlot: s.ParentSlot, + Version: s.Version, + ReferenceTick: s.ReferenceTick, + } + batch, nextData, nextCode, err := gen.makeShredsFromData( + leader, + bytes, + isLastInSlot, + chainedMerkleRoot, + nextShredIndex, + nextCodeIndex, + ) + if err != nil { + return shredPackets{}, nextShredIndex, nextCodeIndex, err + } + return batch, nextData, nextCode, nil +} diff --git a/pkg/turbine/component_test.go b/pkg/turbine/component_test.go index 64b05760c..2617715be 100644 --- a/pkg/turbine/component_test.go +++ b/pkg/turbine/component_test.go @@ -131,6 +131,36 @@ func TestShredEntryBatchRoundTrip(t *testing.T) { require.Equal(t, entry.NumHashes, components[0].EntryBatch[0].NumHashes) } +func TestShredMultiFECEntryBatchRoundTrip(t *testing.T) { + leader := testLeader(t) + entries := make([]turbine.Entry, 1300) + for i := range entries { + entries[i] = turbine.Entry{NumHashes: 1, Hash: solana.Hash{byte(i), byte(i >> 8)}} + } + component, err := turbine.NewEntryBatch(entries) + require.NoError(t, err) + + shredder := turbine.Shredder{Slot: 100, ParentSlot: 99, Version: 42, ReferenceTick: 63} + batch, _, _, err := shredder.MakeMerkleShredsFromComponent( + leader, component, true, solana.Hash{}, 0, 0, + ) + require.NoError(t, err) + require.Greater(t, len(batch.DataShreds), 32) + for i, shred := range batch.DataShreds[:len(batch.DataShreds)-1] { + require.False(t, shred.DataComplete(), "intermediate data shred %d ended the component", i) + } + require.True(t, batch.DataShreds[len(batch.DataShreds)-1].DataComplete()) + require.True(t, batch.DataShreds[len(batch.DataShreds)-1].LastInSlot()) + + components, err := turbine.DecodeComponentsFromDataShreds(batch.DataShreds) + require.NoError(t, err) + require.Len(t, components, 1) + require.Len(t, components[0].EntryBatch, len(entries)) + for i := range entries { + require.Equal(t, entries[i].Hash, components[0].EntryBatch[i].Hash) + } +} + func TestShredBlockHeaderMarkerRoundTrip(t *testing.T) { leader := testLeader(t) parentID := solana.Hash{8} diff --git a/pkg/turbine/generate.go b/pkg/turbine/generate.go index 14626e214..cef43544f 100644 --- a/pkg/turbine/generate.go +++ b/pkg/turbine/generate.go @@ -4,6 +4,7 @@ import ( "crypto/ed25519" "encoding/binary" "fmt" + "sync" "github.com/gagliardetto/solana-go" "github.com/klauspost/reedsolomon" @@ -14,6 +15,21 @@ const ( proofEntriesFor32x32 = 6 ) +var erasureEncoderPool sync.Pool + +func acquireErasureEncoder() (reedsolomon.Encoder, error) { + if encoder := erasureEncoderPool.Get(); encoder != nil { + return encoder.(reedsolomon.Encoder), nil + } + return reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) +} + +func releaseErasureEncoder(encoder reedsolomon.Encoder) { + if encoder != nil { + erasureEncoderPool.Put(encoder) + } +} + // ShredGenerator builds merkle FEC shreds from a serialized byte buffer. type ShredGenerator struct { Slot uint64 @@ -22,6 +38,14 @@ type ShredGenerator struct { ReferenceTick uint8 } +// shredPackets retains the roots already computed during generation, in FEC +// order, so broadcast can commit to them without parsing the packets again. +type shredPackets struct { + packets [][]byte + fecSetRoots []solana.Hash + chainedMerkleRoot solana.Hash +} + func dataCapacity(proofSize uint8, resigned bool) int { capacity := dataPayloadSize - dataHeaderSize - merkleRootSize - int(proofSize)*merkleProofEntrySize if resigned { @@ -66,9 +90,31 @@ func (g *ShredGenerator) MakeShredsFromData( nextShredIndex uint32, nextCodeIndex uint32, ) ([][]byte, solana.Hash, uint32, uint32, error) { + batch, nextData, nextCode, err := g.makeShredsFromData( + leader, data, isLastInSlot, chainedMerkleRoot, nextShredIndex, nextCodeIndex, + ) + return batch.packets, batch.chainedMerkleRoot, nextData, nextCode, err +} + +func (g *ShredGenerator) makeShredsFromData( + leader solana.PrivateKey, + data []byte, + isLastInSlot bool, + chainedMerkleRoot solana.Hash, + nextShredIndex uint32, + nextCodeIndex uint32, +) (shredPackets, uint32, uint32, error) { if g.Slot < g.ParentSlot || g.Slot-g.ParentSlot > uint64(^uint16(0)) { - return nil, solana.Hash{}, nextShredIndex, nextCodeIndex, fmt.Errorf("invalid parent slot %d for slot %d", g.ParentSlot, g.Slot) + return shredPackets{}, nextShredIndex, nextCodeIndex, fmt.Errorf("invalid parent slot %d for slot %d", g.ParentSlot, g.Slot) + } + // The 32+32 coding matrix is invariant across every FEC set in this + // operation. Building it requires a Vandermonde inversion, so retain the + // encoder for the whole payload rather than reconstructing it per set. + encoder, err := acquireErasureEncoder() + if err != nil { + return shredPackets{}, nextShredIndex, nextCodeIndex, err } + defer releaseErasureEncoder(encoder) proofSize := uint8(proofEntriesFor32x32) unsignedCap := dataCapacity(proofSize, false) signedCap := dataCapacity(proofSize, true) @@ -92,6 +138,7 @@ func (g *ShredGenerator) MakeShredsFromData( } var packets [][]byte + var fecSetRoots []solana.Hash dataIndex := nextShredIndex codeIndex := nextCodeIndex chainedRoot := chainedMerkleRoot @@ -99,42 +146,53 @@ func (g *ShredGenerator) MakeShredsFromData( for len(unsignedData) >= unsignedBatch { batch := unsignedData[:unsignedBatch] unsignedData = unsignedData[unsignedBatch:] - batchPackets, root, err := g.makeFECBatch(leader, batch, unsignedCap, proofSize, false, parentOffset, flags, false, chainedRoot, dataIndex, codeIndex) + // DATA_COMPLETE marks the end of the serialized component, not the end + // of every FEC set. A full unsigned batch is complete only when no + // unsigned remainder or signed-last batch follows it. + dataComplete := len(unsignedData) == 0 && len(signedData) == 0 + batchPackets, root, err := g.makeFECBatch(encoder, leader, batch, unsignedCap, proofSize, false, parentOffset, flags, dataComplete, false, chainedRoot, dataIndex, codeIndex) if err != nil { - return nil, solana.Hash{}, dataIndex, codeIndex, err + return shredPackets{}, dataIndex, codeIndex, err } packets = append(packets, batchPackets...) + fecSetRoots = append(fecSetRoots, root) chainedRoot = root dataIndex += dataShredsPerFECBlock codeIndex += codingShredsPerFECBlock } if len(unsignedData) > 0 || (len(packets) == 0 && !isLastInSlot) { - batchPackets, root, err := g.makeFECBatch(leader, unsignedData, unsignedCap, proofSize, false, parentOffset, flags, false, chainedRoot, dataIndex, codeIndex) + dataComplete := len(signedData) == 0 + batchPackets, root, err := g.makeFECBatch(encoder, leader, unsignedData, unsignedCap, proofSize, false, parentOffset, flags, dataComplete, false, chainedRoot, dataIndex, codeIndex) if err != nil { - return nil, solana.Hash{}, dataIndex, codeIndex, err + return shredPackets{}, dataIndex, codeIndex, err } packets = append(packets, batchPackets...) + fecSetRoots = append(fecSetRoots, root) chainedRoot = root dataIndex += dataShredsPerFECBlock codeIndex += codingShredsPerFECBlock } if len(signedData) > 0 || (len(packets) == 0 && isLastInSlot) { - batchPackets, root, err := g.makeFECBatch(leader, signedData, signedCap, proofSize, true, parentOffset, flags, isLastInSlot, chainedRoot, dataIndex, codeIndex) + batchPackets, root, err := g.makeFECBatch(encoder, leader, signedData, signedCap, proofSize, true, parentOffset, flags, true, isLastInSlot, chainedRoot, dataIndex, codeIndex) if err != nil { - return nil, solana.Hash{}, dataIndex, codeIndex, err + return shredPackets{}, dataIndex, codeIndex, err } packets = append(packets, batchPackets...) + fecSetRoots = append(fecSetRoots, root) chainedRoot = root dataIndex += dataShredsPerFECBlock codeIndex += codingShredsPerFECBlock } - return packets, chainedRoot, dataIndex, codeIndex, nil + return shredPackets{ + packets: packets, fecSetRoots: fecSetRoots, chainedMerkleRoot: chainedRoot, + }, dataIndex, codeIndex, nil } func (g *ShredGenerator) makeFECBatch( + encoder reedsolomon.Encoder, leader solana.PrivateKey, data []byte, dataCap int, @@ -142,6 +200,7 @@ func (g *ShredGenerator) makeFECBatch( resigned bool, parentOffset uint16, flags byte, + dataComplete bool, isLastInSlot bool, chainedMerkleRoot solana.Hash, dataIndex uint32, @@ -196,11 +255,11 @@ func (g *ShredGenerator) makeFECBatch( dataPackets[i][dataFlagsOffset] |= shredFlagLastShredInSlot break } - } else if len(dataPackets) > 0 { + } else if dataComplete && len(dataPackets) > 0 { dataPackets[len(dataPackets)-1][dataFlagsOffset] |= shredFlagDataComplete } - root, err := finishErasureBatch(leader, allPackets, chainedMerkleRoot, proofSize, resigned) + root, err := finishErasureBatch(encoder, leader, allPackets, chainedMerkleRoot, proofSize, resigned) if err != nil { return nil, solana.Hash{}, err } @@ -208,101 +267,70 @@ func (g *ShredGenerator) makeFECBatch( } func finishErasureBatch( + encoder reedsolomon.Encoder, leader solana.PrivateKey, packets [][]byte, chainedMerkleRoot solana.Hash, proofSize uint8, resigned bool, ) (solana.Hash, error) { - encoder, err := reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) + if len(packets) != dataShredsPerFECBlock+codingShredsPerFECBlock { + return solana.Hash{}, fmt.Errorf("invalid FEC packet count %d", len(packets)) + } + dataCap, err := merkleCapacity(dataPayloadSize, dataHeaderSize, proofSize, true, resigned) if err != nil { return solana.Hash{}, err } + codeCap, err := merkleCapacity(codingPayloadSize, codingHeaderSize, proofSize, true, resigned) + if err != nil { + return solana.Hash{}, err + } + dataVariant := chainedDataVariant(proofSize, resigned) + codeVariant := chainedCodeVariant(proofSize, resigned) + // These packets were constructed immediately above, so retain direct views + // of their erasure regions. ParseShred is intentionally a defensive, + // owning parser for untrusted network packets; using it here would allocate + // and copy every packet several times only to copy the same bytes back. shards := make([][]byte, len(packets)) for i, packet := range packets { - shred, err := ParseShred(packet) - if err != nil { - return solana.Hash{}, fmt.Errorf("parse batch shred %d: %w", i, err) + if i < dataShredsPerFECBlock { + if len(packet) < dataPayloadSize || packet[shredVariantOffset] != dataVariant { + return solana.Hash{}, fmt.Errorf("invalid generated data shred %d", i) + } + shards[i] = packet[shredSignatureSize : dataHeaderSize+dataCap] + continue } - shard, err := shred.erasureShard() - if err != nil { - return solana.Hash{}, fmt.Errorf("erasure shard %d: %w", i, err) + if len(packet) < codingPayloadSize || packet[shredVariantOffset] != codeVariant { + return solana.Hash{}, fmt.Errorf("invalid generated coding shred %d", i-dataShredsPerFECBlock) } - shards[i] = shard + shards[i] = packet[codingHeaderSize : codingHeaderSize+codeCap] } if err := encoder.Encode(shards); err != nil { return solana.Hash{}, fmt.Errorf("reed-solomon encode: %w", err) } for i, packet := range packets { - shred, err := ParseShred(packet) - if err != nil { - return solana.Hash{}, err - } - proofSizeInfo, chained, resignedFlag, ok := merkleVariantInfo(shred.Variant) - if !ok { - return solana.Hash{}, ErrUnsupportedShred - } - _ = proofSizeInfo - _ = chained - _ = resignedFlag - - capacity, err := merkleCapacity(len(packet), dataHeaderSize, proofSize, true, resigned) - if shred.Type == ShredTypeCode { - capacity, err = merkleCapacity(len(packet), codingHeaderSize, proofSize, true, resigned) - } - if err != nil { - return solana.Hash{}, err - } - rootOffset := dataHeaderSize + capacity - if shred.Type == ShredTypeCode { - rootOffset = codingHeaderSize + capacity + rootOffset := dataHeaderSize + dataCap + if i >= dataShredsPerFECBlock { + rootOffset = codingHeaderSize + codeCap } copy(packet[rootOffset:rootOffset+merkleRootSize], chainedMerkleRoot[:]) - - if shred.Type == ShredTypeCode { - start := codingHeaderSize - end := start + capacity - copy(packet[start:end], shards[i]) - } else { - start := shredSignatureSize - end := dataHeaderSize + capacity - copy(packet[start:end], shards[i]) - } } - nodes, err := buildMerkleTree(packets) - if err != nil { - return solana.Hash{}, err - } + nodes := buildGeneratedMerkleTree(packets, dataCap, codeCap) root := nodes[len(nodes)-1] sig := ed25519.Sign(ed25519.PrivateKey(leader), root[:]) - for _, packet := range packets { + for i, packet := range packets { copy(packet[shredSignatureOffset:shredSignatureSize], sig) - shred, err := ParseShred(packet) - if err != nil { - return solana.Hash{}, err - } - leafIndex, err := shred.merkleLeafIndex() - if err != nil { - return solana.Hash{}, err - } - proof := makeMerkleProof(nodes, leafIndex, len(packets)) - capacity, err := merkleCapacity(len(packet), dataHeaderSize, proofSize, true, resigned) - if shred.Type == ShredTypeCode { - capacity, err = merkleCapacity(len(packet), codingHeaderSize, proofSize, true, resigned) - } - if err != nil { - return solana.Hash{}, err - } - proofOffset := dataHeaderSize + capacity + merkleRootSize - if shred.Type == ShredTypeCode { - proofOffset = codingHeaderSize + capacity + merkleRootSize + proofOffset := dataHeaderSize + dataCap + merkleRootSize + if i >= dataShredsPerFECBlock { + proofOffset = codingHeaderSize + codeCap + merkleRootSize } - for j, entry := range proof { - copy(packet[proofOffset+j*merkleProofEntrySize:], entry[:]) + proofEntries := writeMerkleProof(packet[proofOffset:], nodes, i, len(packets)) + if proofEntries != int(proofSize) { + return solana.Hash{}, fmt.Errorf("generated merkle proof has %d entries, want %d", proofEntries, proofSize) } if resigned { retransmitOffset := proofOffset + int(proofSize)*merkleProofEntrySize @@ -312,6 +340,54 @@ func finishErasureBatch( return root, nil } +// buildGeneratedMerkleTree hashes the fixed packet order emitted by +// makeFECBatch: 32 data shreds followed by 32 coding shreds. Callers must have +// already validated the packet sizes and variants in finishErasureBatch. +func buildGeneratedMerkleTree(packets [][]byte, dataCap, codeCap int) []solana.Hash { + leaves := make([]solana.Hash, len(packets)) + for i, packet := range packets { + end := dataHeaderSize + dataCap + merkleRootSize + if i >= dataShredsPerFECBlock { + end = codingHeaderSize + codeCap + merkleRootSize + } + leaves[i] = merkleHashLeaf(packet[shredSignatureSize:end]) + } + + nodes := make([]solana.Hash, 0, merkleTreeSize(len(leaves))) + nodes = append(nodes, leaves...) + for size := len(leaves); size > 1; size = (size + 1) >> 1 { + offset := len(nodes) - size + for index := offset; index < offset+size; index += 2 { + other := index + 1 + if other >= offset+size { + other = offset + size - 1 + } + nodes = append(nodes, merkleHashNode(nodes[index][:merkleProofEntrySize], nodes[other][:merkleProofEntrySize])) + } + } + return nodes +} + +// writeMerkleProof writes the truncated sibling hashes directly into a packet. +// The generated FEC tree has fixed depth, so materializing a temporary proof +// slice for every one of its 64 packets only adds allocator and copy traffic. +func writeMerkleProof(dst []byte, nodes []solana.Hash, index, size int) int { + entries := 0 + offset := 0 + for size > 1 { + sibling := index ^ 1 + if sibling >= size { + sibling = size - 1 + } + copy(dst[entries*merkleProofEntrySize:], nodes[offset+sibling][:merkleProofEntrySize]) + entries++ + offset += size + size = (size + 1) >> 1 + index >>= 1 + } + return entries +} + func buildMerkleTree(packets [][]byte) ([]solana.Hash, error) { leaves := make([]solana.Hash, len(packets)) for i, packet := range packets { diff --git a/pkg/turbine/generate_bench_test.go b/pkg/turbine/generate_bench_test.go new file mode 100644 index 000000000..2301eefaa --- /dev/null +++ b/pkg/turbine/generate_bench_test.go @@ -0,0 +1,242 @@ +package turbine + +import ( + "crypto/ed25519" + "crypto/sha256" + "encoding/hex" + "fmt" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/gagliardetto/solana-go" + "github.com/klauspost/reedsolomon" +) + +const ( + benchmarkTransactionCount = 50_000 + benchmarkTransactionBytes = 1_232 +) + +var ( + benchmarkEncoderSink reedsolomon.Encoder + benchmarkPacketsSink [][]byte + benchmarkRootSink solana.Hash + benchmarkByteSink byte +) + +func benchmarkLeaderKey() solana.PrivateKey { + var seed [ed25519.SeedSize]byte + for i := range seed { + seed[i] = byte(i + 1) + } + return solana.PrivateKey(ed25519.NewKeyFromSeed(seed[:])) +} + +func benchmarkPayload(size int) []byte { + payload := make([]byte, size) + var state uint64 = 0x9e3779b97f4a7c15 + for i := range payload { + // A deterministic, non-zero corpus avoids accidentally benchmarking a + // special all-zero input while keeping fixture construction out of the + // timed region. + state ^= state << 7 + state ^= state >> 9 + state ^= state << 8 + payload[i] = byte(state) + } + return payload +} + +// BenchmarkReedSolomonEncode32x32 isolates the arithmetic kernel used by one +// unsigned 32+32 chained FEC set. Encoder construction, shred parsing, Merkle +// hashing, signing, and packet copies are intentionally outside this result. +func BenchmarkReedSolomonEncode32x32(b *testing.B) { + const shardBytes = 987 // unsigned chained 32+32 shreds with proof size 6 + encoder, err := reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) + if err != nil { + b.Fatal(err) + } + shards := make([][]byte, dataShredsPerFECBlock+codingShredsPerFECBlock) + for i := range shards { + shards[i] = make([]byte, shardBytes) + if i < dataShredsPerFECBlock { + copy(shards[i], benchmarkPayload(shardBytes)) + } + } + + b.SetBytes(dataShredsPerFECBlock * shardBytes) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if err := encoder.Encode(shards); err != nil { + b.Fatal(err) + } + } + benchmarkByteSink = shards[len(shards)-1][shardBytes-1] +} + +// BenchmarkReedSolomonNew32x32 measures work that finishErasureBatch currently +// repeats for every FEC set even though the 32+32 shape never changes. +func BenchmarkReedSolomonNew32x32(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + encoder, err := reedsolomon.New(dataShredsPerFECBlock, codingShredsPerFECBlock) + if err != nil { + b.Fatal(err) + } + benchmarkEncoderSink = encoder + } +} + +// BenchmarkMakeShredsFromData reports the complete current generator cost, +// including packet construction, Reed-Solomon coding, chained Merkle trees, +// one Ed25519 signature per FEC set, and proof materialization. +func BenchmarkMakeShredsFromData(b *testing.B) { + const blockBytes = benchmarkTransactionCount * benchmarkTransactionBytes + cases := []struct { + name string + size int + isLastInSlot bool + }{ + {name: "one-unsigned-fec", size: dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false)}, + {name: "block-50000x1232", size: blockBytes, isLastInSlot: true}, + } + + for _, tc := range cases { + b.Run(tc.name, func(b *testing.B) { + leader := benchmarkLeaderKey() + payload := benchmarkPayload(tc.size) + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1} + + b.SetBytes(int64(tc.size)) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + packets, root, _, _, err := gen.MakeShredsFromData(leader, payload, tc.isLastInSlot, solana.Hash{}, 0, 0) + if err != nil { + b.Fatal(err) + } + benchmarkPacketsSink = packets + benchmarkRootSink = root + } + b.StopTimer() + if len(benchmarkPacketsSink) == 0 || len(benchmarkPacketsSink)%64 != 0 { + b.Fatalf("unexpected packet count %d", len(benchmarkPacketsSink)) + } + b.ReportMetric(float64(len(benchmarkPacketsSink)), "packets/op") + b.ReportMetric(float64(len(benchmarkPacketsSink)/64), "FEC-sets/op") + if tc.size == blockBytes { + b.ReportMetric(benchmarkTransactionCount, "transactions/op") + } + }) + } +} + +// BenchmarkMakeShreds50000TargetBatches1232 models the producer's target-sized +// component stream without retaining a multi-gigabyte output. It measures only +// the 61.6 MB transaction payload; entry framing is deliberately outside this +// erasure-coding benchmark. +func BenchmarkMakeShreds50000TargetBatches1232(b *testing.B) { + const inputBytes = benchmarkTransactionCount * benchmarkTransactionBytes + leader := benchmarkLeaderKey() + payload := benchmarkPayload(inputBytes) + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1} + + b.SetBytes(inputBytes) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + var ( + root solana.Hash + dataIndex uint32 + codeIndex uint32 + ) + var ( + offset int + components int + packetCount int + fecSetCount int + ) + for offset < len(payload) { + end := min(offset+costmodel.DefaultTargetBatchBytes, len(payload)) + packets, nextRoot, nextData, nextCode, err := gen.MakeShredsFromData( + leader, + payload[offset:end], + end == len(payload), + root, + dataIndex, + codeIndex, + ) + if err != nil { + b.Fatal(err) + } + benchmarkPacketsSink = packets + components++ + packetCount += len(packets) + fecSetCount += len(packets) / (dataShredsPerFECBlock + codingShredsPerFECBlock) + root, dataIndex, codeIndex = nextRoot, nextData, nextCode + offset = end + } + b.ReportMetric(float64(components), "components/op") + b.ReportMetric(float64(fecSetCount), "FEC-sets/op") + b.ReportMetric(float64(packetCount), "packets/op") + benchmarkRootSink = root + } + b.StopTimer() + b.ReportMetric(benchmarkTransactionCount, "transactions/op") +} + +func TestBenchmarkBlockPayloadAccounting(t *testing.T) { + const blockBytes = benchmarkTransactionCount * benchmarkTransactionBytes + unsignedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + unsignedBytes := blockBytes - signedBatch + unsignedFECs := (unsignedBytes + unsignedBatch - 1) / unsignedBatch + totalFECs := unsignedFECs + 1 + if totalFECs != 2000 { + t.Fatalf("50k x 1232 payload maps to %d FEC sets, want 2000 (%s)", totalFECs, fmt.Sprintf("%d bytes", blockBytes)) + } +} + +func TestProducerTargetMatchesTwoTypicalFECPayloads(t *testing.T) { + want := 2 * dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + if costmodel.DefaultTargetBatchBytes != want { + t.Fatalf("producer target = %d, want two typical FEC payloads = %d", costmodel.DefaultTargetBatchBytes, want) + } +} + +func TestMakeShredsFromDataStableBytes(t *testing.T) { + unsignedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + tests := []struct { + name string + size int + isLastInSlot bool + want string + }{ + {name: "unsigned-one-fec", size: unsignedBatch, want: "f9334f1835240df21d4b48a09f35b3ff90578122d30d0527608c10d38d0911f7"}, + {name: "signed-one-fec", size: signedBatch, isLastInSlot: true, want: "bfa398c445509c5e1345553bbe84fea04e86001caf441008b4014d07c6e36ccd"}, + {name: "two-unsigned-one-signed", size: 2*unsignedBatch + signedBatch, isLastInSlot: true, want: "dffae840e4c247680b5e1667747a63138872a0080c51a6fcf4cc5002eb7778ac"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gen := ShredGenerator{Slot: 10, ParentSlot: 9, Version: 1, ReferenceTick: 17} + packets, _, _, _, err := gen.MakeShredsFromData( + benchmarkLeaderKey(), benchmarkPayload(tt.size), tt.isLastInSlot, + solana.Hash{3}, 7, 11, + ) + if err != nil { + t.Fatal(err) + } + h := sha256.New() + for _, packet := range packets { + _, _ = h.Write(packet) + } + got := hex.EncodeToString(h.Sum(nil)) + if got != tt.want { + t.Fatalf("packet digest %s, want %s", got, tt.want) + } + }) + } +} diff --git a/pkg/turbine/generate_test.go b/pkg/turbine/generate_test.go index 083133954..64c5d1091 100644 --- a/pkg/turbine/generate_test.go +++ b/pkg/turbine/generate_test.go @@ -4,6 +4,7 @@ import ( "bytes" "crypto/ed25519" "encoding/binary" + "fmt" "testing" "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" @@ -174,6 +175,46 @@ func TestMakeShredsFromDataRoundTrip(t *testing.T) { } } +func TestGeneratedFECRootsMatchPacketProofs(t *testing.T) { + unsignedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, false) + signedBatch := dataShredsPerFECBlock * dataCapacity(proofEntriesFor32x32, true) + for _, last := range []bool{false, true} { + for _, size := range []int{0, 1, signedBatch, signedBatch + 1, unsignedBatch, unsignedBatch + 1, 2 * unsignedBatch, 2*unsignedBatch + signedBatch} { + t.Run(fmt.Sprintf("last=%t/bytes=%d", last, size), func(t *testing.T) { + gen := ShredGenerator{Slot: 100, ParentSlot: 99, Version: 7, ReferenceTick: 17} + parentRoot := solana.Hash{5} + leader := testShredLeader(t) + batch, nextData, nextCode, err := gen.makeShredsFromData( + leader, benchmarkPayload(size), last, parentRoot, 7, 11, + ) + require.NoError(t, err) + require.NotEmpty(t, batch.fecSetRoots) + require.Len(t, batch.packets, len(batch.fecSetRoots)*(dataShredsPerFECBlock+codingShredsPerFECBlock)) + require.Equal(t, uint32(7+len(batch.fecSetRoots)*dataShredsPerFECBlock), nextData) + require.Equal(t, uint32(11+len(batch.fecSetRoots)*codingShredsPerFECBlock), nextCode) + require.Equal(t, batch.fecSetRoots[len(batch.fecSetRoots)-1], batch.chainedMerkleRoot) + for i, packet := range batch.packets { + fec := i / (dataShredsPerFECBlock + codingShredsPerFECBlock) + shred, err := ParseShred(packet) + require.NoError(t, err) + root, err := shred.MerkleRoot() + require.NoError(t, err) + require.Equal(t, batch.fecSetRoots[fec], root) + require.Equal(t, uint32(7+fec*dataShredsPerFECBlock), shred.FECSetIndex) + previousRoot := parentRoot + if fec > 0 { + previousRoot = batch.fecSetRoots[fec-1] + } + embeddedRoot, err := shred.EmbeddedChainedMerkleRoot() + require.NoError(t, err) + require.Equal(t, previousRoot, embeddedRoot) + require.NoError(t, shred.VerifySignature(leader.PublicKey())) + } + }) + } + } +} + func TestMakeShredsFromAlpenglowBlock(t *testing.T) { leader := testShredLeader(t) gen := ShredGenerator{ @@ -184,10 +225,10 @@ func TestMakeShredsFromAlpenglowBlock(t *testing.T) { } var ( - chainedRoot = solana.Hash{5} - nextData uint32 = 0 - nextCode uint32 = 0 - allDataShreds []*Shred + chainedRoot = solana.Hash{5} + nextData uint32 = 0 + nextCode uint32 = 0 + allDataShreds []*Shred ) for _, component := range buildAlpenglowSlot(t) { packets, root, newData, newCode, err := gen.MakeShredsFromData( diff --git a/pkg/turbine/internal/rsrecover/doc.go b/pkg/turbine/internal/rsrecover/doc.go new file mode 100644 index 000000000..adac8aad8 --- /dev/null +++ b/pkg/turbine/internal/rsrecover/doc.go @@ -0,0 +1,8 @@ +// Package rsrecover contains experimental, fixed-shape Reed-Solomon recovery +// plans for Solana's 32 data + 32 coding shred FEC sets. +// +// Nothing in the production turbine assembler imports this package. It exists +// to measure two workload-specific questions before either policy is wired in: +// near-tip latency for one missing data shred, and catch-up throughput for a +// subset of missing data shreds. +package rsrecover diff --git a/pkg/turbine/internal/rsrecover/recover.go b/pkg/turbine/internal/rsrecover/recover.go new file mode 100644 index 000000000..6a16e91e9 --- /dev/null +++ b/pkg/turbine/internal/rsrecover/recover.go @@ -0,0 +1,599 @@ +package rsrecover + +import ( + "errors" + "fmt" + "math/bits" + "sync" + + "github.com/klauspost/reedsolomon" +) + +const ( + DataShards = 32 + CodingShards = 32 + TotalShards = DataShards + CodingShards + + cauchyGamma = byte(0xa5) +) + +var ( + ErrInvalidPattern = errors.New("invalid erasure pattern") + ErrPatternChanged = errors.New("availability differs from prepared plan") + ErrInvalidBuffers = errors.New("invalid recovery buffers") + + gfLog [256]byte + gfExp [512]byte + + allCodingEncoderOnce sync.Once + allCodingEncoder reedsolomon.Encoder + allCodingEncoderErr error + + // oneDataCoefficientRows[missing][coding] contains one coefficient for + // each data input followed by the selected coding input. Every one of the + // fixed 32x32 rows is exhaustively differential-tested against the general + // Reed-Solomon decoder. + oneDataCoefficientRows [DataShards][CodingShards][DataShards + 1]byte +) + +func init() { + value := uint16(1) + for exponent := 0; exponent < 255; exponent++ { + gfExp[exponent] = byte(value) + gfLog[byte(value)] = byte(exponent) + value <<= 1 + if value&0x100 != 0 { + value ^= 0x11d + } + } + for exponent := 255; exponent < len(gfExp); exponent++ { + gfExp[exponent] = gfExp[exponent-255] + } + for missing := 0; missing < DataShards; missing++ { + for coding := 0; coding < CodingShards; coding++ { + // If c = sum(a_i*d_i), then the missing d_m is + // inv(a_m) * (c + sum(i != m, a_i*d_i)) over GF(2^8). + coefficientInv := reedsolomon.Inv(codingCoefficient(coding, missing)) + for data := 0; data < DataShards; data++ { + if data != missing { + oneDataCoefficientRows[missing][coding][data] = gfMul( + coefficientInv, + codingCoefficient(coding, data), + ) + } + } + oneDataCoefficientRows[missing][coding][DataShards] = coefficientInv + } + } +} + +// OneDataPlan recovers exactly one absent data shard from the other 31 data +// shards and one available coding shard. It avoids a general 32x32 decode +// matrix inversion. The plan is immutable and safe for concurrent execution +// when callers provide independent destinations. +type OneDataPlan struct { + presence uint64 + missing uint8 + sources [DataShards]uint8 + coefficients [DataShards]byte +} + +// DataSubsetPlan recovers every absent data shard using the present data and +// the lowest-indexed coding rows required to reach the 32-shard threshold. +// Setup solves only the m x m system induced by the missing data columns. +type DataSubsetPlan struct { + presence uint64 + missing []uint8 + sources [DataShards]uint8 + weights [][]byte +} + +// AllCodingPlan recovers all 32 data rows from all 32 coding rows. For the +// fixed Solana matrix C*C=I, so the package's optimized encoder can apply C a +// second time instead of constructing a decode matrix. +type AllCodingPlan struct { + presence uint64 + encoder reedsolomon.Encoder +} + +// Presence reports which of the 64 input shards are non-empty. A zero-length +// shard is absent, matching reedsolomon.ReconstructSome. +func Presence(shards [][]byte) (uint64, error) { + if len(shards) != TotalShards { + return 0, fmt.Errorf("%w: got %d shards, want %d", ErrInvalidBuffers, len(shards), TotalShards) + } + var mask uint64 + for index, shard := range shards { + if len(shard) != 0 { + mask |= uint64(1) << index + } + } + return mask, nil +} + +// PrepareRecoverOneData constructs the direct coefficient row for a pattern +// with exactly one missing data shard. Additional coding shards may be present; +// the lowest-indexed one is selected deterministically. +func PrepareRecoverOneData(presence uint64, missingDataIndex int) (OneDataPlan, error) { + if missingDataIndex < 0 || missingDataIndex >= DataShards { + return OneDataPlan{}, fmt.Errorf("%w: missing data index %d", ErrInvalidPattern, missingDataIndex) + } + for index := 0; index < DataShards; index++ { + present := presence&(uint64(1)<= DataShards { + return fmt.Errorf("%w: missing data index %d", ErrInvalidPattern, missingDataIndex) + } + const dataMask = uint64(1)<> DataShards) + if codingMask == 0 { + return fmt.Errorf("%w: no coding shard is available", ErrInvalidPattern) + } + shardSize, err := validateExecution(presence, shards, [][]byte{dst}) + if err != nil { + return err + } + if len(dst) != shardSize { + return fmt.Errorf("%w: destination has %d bytes, want %d", ErrInvalidBuffers, len(dst), shardSize) + } + + codingPosition := bits.TrailingZeros32(codingMask) + row := &oneDataCoefficientRows[missingDataIndex][codingPosition] + var lowLevel reedsolomon.LowLevel + first := true + for dataIndex := 0; dataIndex < DataShards; dataIndex++ { + coefficient := row[dataIndex] + if coefficient == 0 { + continue + } + if first { + lowLevel.GalMulSlice(coefficient, shards[dataIndex], dst) + first = false + } else { + lowLevel.GalMulSliceXor(coefficient, shards[dataIndex], dst) + } + } + lowLevel.GalMulSliceXor(row[DataShards], shards[DataShards+codingPosition], dst) + return nil +} + +// PrepareRecoverDataSubset constructs direct output rows for all absent data +// shards. It first inverts only the reduced m x m coding/data matrix, then +// expands those rows over exactly 32 selected input shards so byte execution +// can be compared fairly with a general decoder. +func PrepareRecoverDataSubset(presence uint64) (DataSubsetPlan, error) { + plan := DataSubsetPlan{presence: presence} + knownData := make([]uint8, 0, DataShards) + for index := 0; index < DataShards; index++ { + if presence&(uint64(1)<>uint((i%8)*8)) ^ byte(i*29+7) + } + digest := sha256.Sum256(append([]byte("mithril-repair-sim-leader-v1"), input[:]...)) + return solana.PrivateKey(ed25519.NewKeyFromSeed(digest[:])) +} + +func deterministicEntries(seed int64, slot uint64, count int) []turbine.Entry { + entries := make([]turbine.Entry, count) + for i := range entries { + material := fmt.Sprintf("mithril-repair-sim-entry-v1:%d:%d:%d", seed, slot, i) + h := sha256.Sum256([]byte(material)) + entries[i] = turbine.Entry{NumHashes: 1, Hash: solana.Hash(h)} + } + return entries +} + +// findEntryCount uses the generator itself as the capacity oracle. This avoids +// duplicating signed-last-FEC payload constants in the harness. +func findEntryCount(cfg LedgerConfig, leader solana.PrivateKey) (int, error) { + lo, hi := 1, cfg.FECsPerSlot*900 + for lo < hi { + mid := lo + (hi-lo)/2 + entries := deterministicEntries(cfg.Seed, cfg.StartSlot, mid) + slot, err := generateSlot(cfg, leader, cfg.StartSlot, cfg.StartSlot-1, entries) + if err != nil { + return 0, err + } + if len(slot.FECs) < cfg.FECsPerSlot { + lo = mid + 1 + } else { + hi = mid + } + } + entries := deterministicEntries(cfg.Seed, cfg.StartSlot, lo) + slot, err := generateSlot(cfg, leader, cfg.StartSlot, cfg.StartSlot-1, entries) + if err != nil { + return 0, err + } + if len(slot.FECs) != cfg.FECsPerSlot { + return 0, fmt.Errorf("cannot derive %d FEC sets within %d entries (got %d)", cfg.FECsPerSlot, hi, len(slot.FECs)) + } + return lo, nil +} + +func generateSlot(cfg LedgerConfig, leader solana.PrivateKey, number, parent uint64, entries []turbine.Entry) (Slot, error) { + component, err := turbine.NewEntryBatch(entries) + if err != nil { + return Slot{}, err + } + shredder := turbine.Shredder{ + Slot: number, + ParentSlot: parent, + Version: cfg.ShredVersion, + ReferenceTick: cfg.ReferenceTick, + } + batch, _, _, err := shredder.MakeMerkleShredsFromComponent( + leader, component, true, solana.Hash{}, 0, 0, + ) + if err != nil { + return Slot{}, err + } + + byFEC := make(map[uint32]*FECSet) + packetByKey := make(map[packetKey][]byte, len(batch.Packets)) + for _, raw := range batch.Packets { + shred, err := turbine.ParseShred(raw) + if err != nil { + return Slot{}, err + } + key := keyForShred(shred) + packetByKey[key] = append([]byte(nil), raw...) + } + data := make(map[uint32]Packet, len(batch.DataShreds)) + var highest uint32 + for _, shred := range append(append([]*turbine.Shred(nil), batch.DataShreds...), batch.CodeShreds...) { + fec := byFEC[shred.FECSetIndex] + if fec == nil { + fec = &FECSet{Index: shred.FECSetIndex} + byFEC[shred.FECSetIndex] = fec + } + packet := Packet{ + Bytes: packetByKey[keyForShred(shred)], + Slot: shred.Slot, + Type: shred.Type, + Index: shred.Index, + FECSetIndex: shred.FECSetIndex, + Position: shred.Position, + } + if shred.Type == turbine.ShredTypeData { + fec.Data = append(fec.Data, packet) + data[shred.Index] = packet + if shred.Index > highest { + highest = shred.Index + } + } else { + fec.Coding = append(fec.Coding, packet) + } + } + fecs := make([]FECSet, 0, len(byFEC)) + for index := uint32(0); len(fecs) < len(byFEC); index += dataShredsPerFEC { + fec := byFEC[index] + if fec == nil { + return Slot{}, fmt.Errorf("non-contiguous FEC sets: missing %d", index) + } + if len(fec.Data) != dataShredsPerFEC || len(fec.Coding) != codeShredsPerFEC { + return Slot{}, fmt.Errorf("FEC %d has %d+%d shreds", index, len(fec.Data), len(fec.Coding)) + } + fecs = append(fecs, *fec) + } + return Slot{Number: number, ParentSlot: parent, Entries: entries, FECs: fecs, Data: data, Highest: highest}, nil +} + +type packetKey struct { + type_ turbine.ShredType + index uint32 + fec uint32 + position uint16 +} + +func keyForShred(shred *turbine.Shred) packetKey { + return packetKey{type_: shred.Type, index: shred.Index, fec: shred.FECSetIndex, position: shred.Position} +} diff --git a/pkg/turbine/repairsim/sim.go b/pkg/turbine/repairsim/sim.go new file mode 100644 index 000000000..c89e867e1 --- /dev/null +++ b/pkg/turbine/repairsim/sim.go @@ -0,0 +1,713 @@ +package repairsim + +import ( + "container/heap" + "errors" + "fmt" + "math/rand" + "os" + "runtime" + "sort" + "time" + + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/turbine" +) + +type Scenario string + +const ( + ScenarioNearTip Scenario = "near-tip" + ScenarioDeepCatchup Scenario = "deep-catchup" +) + +type Availability string + +const ( + AvailabilityComplete Availability = "complete" + AvailabilityNearLoss Availability = "near-loss" + AvailabilitySparse Availability = "sparse" + AvailabilityMixed Availability = "mixed" +) + +// Config controls the deterministic network and local starting state. +type Config struct { + Scenario Scenario `json:"scenario"` + Availability Availability `json:"availability"` + RepairEnabled bool `json:"repair_enabled"` + RepairLatency time.Duration `json:"repair_latency_ns"` + RepairJitter time.Duration `json:"repair_jitter_ns"` + PacketLoss float64 `json:"packet_loss"` + DuplicateProbability float64 `json:"duplicate_probability"` + BandwidthBytesPerSec int64 `json:"bandwidth_bytes_per_sec"` + MaxConcurrent int `json:"max_concurrent_requests"` + MaxRequestSlots int `json:"max_request_slots"` + MaxMissingPerSlot int `json:"max_missing_per_slot"` + CorruptResponses int `json:"corrupt_responses"` + NaturalLateShreds bool `json:"natural_late_shreds"` + CollectTrace bool `json:"collect_trace"` + Seed int64 `json:"seed"` + SpoolDir string `json:"spool_dir,omitempty"` + SpoolMaxBytes int64 `json:"spool_max_bytes"` +} + +// DefaultConfig returns a deterministic starting point for a scenario. +func DefaultConfig(scenario Scenario) Config { + cfg := Config{ + Scenario: scenario, + RepairEnabled: true, + RepairLatency: 20 * time.Millisecond, + RepairJitter: 2 * time.Millisecond, + DuplicateProbability: 0.02, + BandwidthBytesPerSec: 100 * 1024 * 1024, + MaxConcurrent: 256, + MaxRequestSlots: 64, + MaxMissingPerSlot: 256, + NaturalLateShreds: scenario == ScenarioNearTip, + CollectTrace: true, + Seed: 1, + SpoolMaxBytes: 1 << 30, + } + if scenario == ScenarioDeepCatchup { + cfg.Availability = AvailabilityMixed + } else { + cfg.Availability = AvailabilityNearLoss + } + return cfg +} + +// TraceEvent is a deterministic logical-time event. CPU durations are kept in +// Result.StageCPU so trace equality does not depend on scheduler noise. +type TraceEvent struct { + Sequence int `json:"sequence"` + AtNanos int64 `json:"at_ns"` + Stage string `json:"stage"` + Slot uint64 `json:"slot,omitempty"` + FECSetIndex uint32 `json:"fec_set_index,omitempty"` + ShredIndex uint32 `json:"shred_index,omitempty"` + ShredType turbine.ShredType `json:"shred_type,omitempty"` + Bytes int `json:"bytes,omitempty"` + Detail string `json:"detail,omitempty"` +} + +type LatencySummary struct { + P50 time.Duration `json:"p50_ns"` + P95 time.Duration `json:"p95_ns"` + P99 time.Duration `json:"p99_ns"` +} + +// Result separates simulated-network time from actual local execution time. +type Result struct { + Scenario Scenario `json:"scenario"` + Availability Availability `json:"availability"` + Slots int `json:"slots"` + CompletedSlots int `json:"completed_slots"` + LogicalElapsed time.Duration `json:"logical_elapsed_ns"` + WallElapsed time.Duration `json:"wall_elapsed_ns"` + TimeToFirstReplayable time.Duration `json:"time_to_first_replayable_ns"` + TimeToFirstRecoveredData time.Duration `json:"time_to_first_recovered_data_ns"` + RepairEligibleToRecovery time.Duration `json:"repair_eligible_to_first_recovery_ns"` + CompletionLatency LatencySummary `json:"completion_latency"` + SlotsPerLogicalSecond float64 `json:"slots_per_logical_second"` + SlotsPerCPUSecond float64 `json:"slots_per_cpu_second"` + RepairRequests uint64 `json:"repair_requests"` + RepairResponses uint64 `json:"repair_responses"` + RepairBytesRequested uint64 `json:"repair_bytes_requested"` + RepairBytesReceived uint64 `json:"repair_bytes_received"` + UsefulNetworkDataShreds uint64 `json:"useful_network_data_shreds"` + LocallyRecoveredDataShreds uint64 `json:"locally_recovered_data_shreds"` + LocallyRecoveredDataBytes uint64 `json:"locally_recovered_data_bytes"` + InitialMissingDataShreds uint64 `json:"initial_missing_data_shreds"` + FractionRecoveredLocally float64 `json:"fraction_missing_recovered_locally"` + FECDecodes uint64 `json:"fec_decodes"` + DuplicateResponses uint64 `json:"duplicate_responses"` + CanceledOrLateResponses uint64 `json:"canceled_or_late_responses"` + LostResponses uint64 `json:"lost_responses"` + RejectedCorruptResponses uint64 `json:"rejected_corrupt_responses"` + ShredSignatureCacheHits uint64 `json:"shred_signature_cache_hits"` + ShredEd25519Verifications uint64 `json:"shred_ed25519_verifications"` + QueueHighWater int `json:"queue_high_water"` + SpoolBytes int64 `json:"spool_bytes"` + SpoolCompleteSlots int `json:"spool_complete_slots"` + DataShredBytesReplayable uint64 `json:"data_shred_bytes_replayable"` + Allocations uint64 `json:"allocations"` + StageCPU map[string]time.Duration `json:"stage_cpu_ns"` + Trace []TraceEvent `json:"trace"` + Limitations []string `json:"limitations"` +} + +// Run executes the virtual network around production parsing, validation, +// repair selection, reconstruction, spool insertion, and block completion. +func Run(ledger *Ledger, cfg Config) (Result, error) { + if ledger == nil || len(ledger.Slots) == 0 { + return Result{}, errors.New("empty ledger") + } + if cfg.Scenario != ScenarioNearTip && cfg.Scenario != ScenarioDeepCatchup { + return Result{}, fmt.Errorf("unsupported scenario %q", cfg.Scenario) + } + if cfg.Availability == "" { + cfg.Availability = DefaultConfig(cfg.Scenario).Availability + } + if cfg.MaxConcurrent <= 0 { + cfg.MaxConcurrent = 1 + } + if cfg.MaxRequestSlots <= 0 { + cfg.MaxRequestSlots = 64 + } + if cfg.MaxMissingPerSlot <= 0 { + cfg.MaxMissingPerSlot = 256 + } + if cfg.SpoolMaxBytes <= 0 { + cfg.SpoolMaxBytes = 1 << 30 + } + + spoolDir := cfg.SpoolDir + if spoolDir == "" { + var err error + spoolDir, err = os.MkdirTemp("", "mithril-repair-sim-") + if err != nil { + return Result{}, err + } + defer os.RemoveAll(spoolDir) + } + spool, err := turbine.OpenShredSpool(spoolDir, cfg.SpoolMaxBytes) + if err != nil { + return Result{}, err + } + defer spool.Close() + + var memBefore runtime.MemStats + runtime.ReadMemStats(&memBefore) + wallStarted := time.Now() + s := &simulation{ + ledger: ledger, + cfg: cfg, + assembler: turbine.NewSlotAssembler(), + spool: spool, + rng: rand.New(rand.NewSource(cfg.Seed)), + firstShred: make(map[uint64]time.Duration), + completed: make(map[uint64]*block.Block), + replayableAt: make(map[uint64]time.Duration), + pending: make(map[repairKey]struct{}), + stageCPU: make(map[string]time.Duration), + } + s.assembler.SetRetentionFloor(ledger.Slots[0].Number) + s.assembler.SetOnComplete(spool.MarkComplete) + s.nextReplaySlot = ledger.Slots[0].Number + + if err := s.seedLocalState(); err != nil { + return Result{}, err + } + if err := s.runDeliveries(); err != nil { + return Result{}, err + } + + var memAfter runtime.MemStats + runtime.ReadMemStats(&memAfter) + _, spoolBytes := spool.Stats() + result := s.result + result.Scenario = cfg.Scenario + result.Availability = cfg.Availability + result.Slots = len(ledger.Slots) + result.CompletedSlots = len(s.completed) + result.LogicalElapsed = s.now + result.WallElapsed = time.Since(wallStarted) + result.LocallyRecoveredDataShreds = s.assembler.RecoveredDataShreds() + result.InitialMissingDataShreds = initialMissingDataShreds(ledger, cfg.Availability) + if result.InitialMissingDataShreds > 0 { + result.FractionRecoveredLocally = float64(result.LocallyRecoveredDataShreds) / float64(result.InitialMissingDataShreds) + } + if len(ledger.Slots[0].FECs) > 0 && len(ledger.Slots[0].FECs[0].Data) > 0 { + result.LocallyRecoveredDataBytes = result.LocallyRecoveredDataShreds * uint64(len(ledger.Slots[0].FECs[0].Data[0].Bytes)) + } + if s.haveRecoverAt { + result.TimeToFirstRecoveredData = s.firstRecoverAt + if s.haveRepairAt && s.firstRecoverAt >= s.firstRepairAt { + result.RepairEligibleToRecovery = s.firstRecoverAt - s.firstRepairAt + } + } + result.SpoolBytes = spoolBytes + result.SpoolCompleteSlots = spool.CompleteSlots() + result.Allocations = memAfter.Mallocs - memBefore.Mallocs + result.StageCPU = s.stageCPU + result.Trace = s.trace + result.ShredSignatureCacheHits, result.ShredEd25519Verifications = s.shredVerifier.Stats() + result.Limitations = []string{ + "remote peers and latency are simulated in process; no UDP/IP stack is measured", + "synthetic entries contain no transactions, so transaction execution is not measured", + "slot offered to replay means SlotAssembler emitted a verified block; replay execution is not run", + "FEC decode start is observed at the AddShredFrom call boundary, not inside the Reed-Solomon library", + } + latencies := make([]time.Duration, 0, len(s.replayableAt)) + for slot, completedAt := range s.replayableAt { + if first, ok := s.firstShred[slot]; ok { + latencies = append(latencies, completedAt-first) + } + } + result.CompletionLatency = summarizeLatencies(latencies) + if len(s.replayableAt) > 0 { + first := ledger.Slots[0].Number + result.TimeToFirstReplayable = s.replayableAt[first] + } + if result.LogicalElapsed > 0 { + result.SlotsPerLogicalSecond = float64(result.CompletedSlots) / result.LogicalElapsed.Seconds() + } + if result.WallElapsed > 0 { + result.SlotsPerCPUSecond = float64(result.CompletedSlots) / result.WallElapsed.Seconds() + } + return result, nil +} + +type simulation struct { + ledger *Ledger + cfg Config + assembler *turbine.SlotAssembler + shredVerifier turbine.ShredSignatureVerifier + spool *turbine.ShredSpool + rng *rand.Rand + now time.Duration + nextWireAt time.Duration + sequence int + trace []TraceEvent + queue deliveryHeap + pending map[repairKey]struct{} + firstShred map[uint64]time.Duration + completed map[uint64]*block.Block + replayableAt map[uint64]time.Duration + nextReplaySlot uint64 + stageCPU map[string]time.Duration + result Result + corruptLeft int + firstRepairAt time.Duration + haveRepairAt bool + firstRecoverAt time.Duration + haveRecoverAt bool +} + +func (s *simulation) seedLocalState() error { + s.corruptLeft = s.cfg.CorruptResponses + packetsBySlot := make([][]Packet, len(s.ledger.Slots)) + for slotIdx := range s.ledger.Slots { + packetsBySlot[slotIdx] = initialPackets(&s.ledger.Slots[slotIdx], s.cfg.Availability) + } + for ordinal := 0; ; ordinal++ { + added := false + for slotIdx := range s.ledger.Slots { + packets := packetsBySlot[slotIdx] + if ordinal >= len(packets) { + continue + } + added = true + if err := s.ingest(packets[ordinal], false, false); err != nil { + return fmt.Errorf("seed slot %d: %w", s.ledger.Slots[slotIdx].Number, err) + } + s.now++ + } + if !added { + break + } + } + if s.cfg.NaturalLateShreds && s.cfg.Availability == AvailabilityNearLoss { + for i := range s.ledger.Slots { + slot := &s.ledger.Slots[i] + if len(slot.FECs) == 0 || len(slot.FECs[0].Data) < 31 { + continue + } + at := s.now + s.cfg.RepairLatency/2 + time.Duration(i)*time.Microsecond + heap.Push(&s.queue, delivery{at: at, sequence: s.sequence, packet: slot.FECs[0].Data[30]}) + s.sequence++ + } + } + return nil +} + +func initialPackets(slot *Slot, availability Availability) []Packet { + var out []Packet + for i := range slot.FECs { + fec := &slot.FECs[i] + switch availability { + case AvailabilityComplete: + out = append(out, fec.Data...) + case AvailabilityNearLoss: + if i%2 == 0 { + out = append(out, fec.Data[:30]...) + out = append(out, fec.Coding[0]) + } else { + out = append(out, fec.Data[:31]...) + } + case AvailabilitySparse: + out = append(out, fec.Data[:2]...) + case AvailabilityMixed: + out = append(out, fec.Data[:16]...) + out = append(out, fec.Coding[:15]...) + } + } + return out +} + +func (s *simulation) runDeliveries() error { + const maxIterations = 10_000_000 + for iterations := 0; len(s.completed) < len(s.ledger.Slots); iterations++ { + if iterations >= maxIterations { + return errors.New("repair simulation exceeded iteration limit") + } + if s.cfg.RepairEnabled { + s.prioritizeHeadWindow() + s.scheduleRequests() + } + if len(s.queue) == 0 { + // Without repair, exhaust the live arrivals and report any remaining + // holes. No more traffic is expected to complete those slots. + if !s.cfg.RepairEnabled { + return nil + } + return fmt.Errorf("repair stalled with %d/%d completed", len(s.completed), len(s.ledger.Slots)) + } + event := heap.Pop(&s.queue).(delivery) + if event.at > s.now { + s.now = event.at + } + if event.primary { + delete(s.pending, event.key) + } + if event.drop { + s.result.LostResponses++ + s.record("repair_response_lost", event.packet, "") + continue + } + if event.duplicate { + s.result.DuplicateResponses++ + } + if event.fromRepair { + s.result.RepairResponses++ + s.result.RepairBytesReceived += uint64(len(event.packet.Bytes)) + } + beforeUseful := s.assembler.UsefulRepairShreds() + if err := s.ingest(event.packet, event.fromRepair, event.corrupt); err != nil { + if event.corrupt && errors.Is(err, turbine.ErrInvalidSignature) { + s.result.RejectedCorruptResponses++ + s.record("repair_response_rejected", event.packet, "invalid signature or Merkle proof") + continue + } + return err + } + if event.fromRepair { + afterUseful := s.assembler.UsefulRepairShreds() + if afterUseful == beforeUseful { + s.result.CanceledOrLateResponses++ + } else { + s.result.UsefulNetworkDataShreds += afterUseful - beforeUseful + } + } + } + return nil +} + +func (s *simulation) prioritizeHeadWindow() { + var head uint64 + found := false + for i := range s.ledger.Slots { + slot := s.ledger.Slots[i].Number + if _, complete := s.completed[slot]; !complete { + head, found = slot, true + break + } + } + if !found { + return + } + end := head + 63 + last := s.ledger.Slots[len(s.ledger.Slots)-1].Number + if end > last { + end = last + } + s.assembler.PrioritizeRepairRange(head, end) +} + +func (s *simulation) scheduleRequests() { + capacity := s.cfg.MaxConcurrent - len(s.pending) + if capacity <= 0 { + return + } + requests := s.assembler.RepairRequests(s.cfg.MaxRequestSlots, s.cfg.MaxMissingPerSlot) + for _, req := range requests { + if !s.haveRepairAt { + s.firstRepairAt = s.now + s.haveRepairAt = true + } + s.recordAt("repair_needed_decision", req.Slot, 0, 0, 0, 0, + fmt.Sprintf("missing=%d need_highest=%t", len(req.MissingDataShreds), req.NeedHighestDataShred)) + slot, ok := s.ledger.Slot(req.Slot) + if !ok { + continue + } + indexes := append([]uint32(nil), req.MissingDataShreds...) + if req.NeedHighestDataShred { + indexes = append(indexes, slot.Highest) + } + seen := make(map[uint32]struct{}, len(indexes)) + for _, index := range indexes { + if capacity == 0 { + return + } + if _, duplicate := seen[index]; duplicate { + continue + } + seen[index] = struct{}{} + packet, ok := slot.Data[index] + if !ok { + continue + } + key := repairKey{slot: req.Slot, index: index} + if _, outstanding := s.pending[key]; outstanding { + continue + } + s.pending[key] = struct{}{} + capacity-- + s.result.RepairRequests++ + s.result.RepairBytesRequested += uint64(len(packet.Bytes)) + s.record("repair_request_enqueue", packet, "data shred") + s.record("repair_request_send", packet, "data shred") + s.scheduleResponse(key, packet) + } + } +} + +func (s *simulation) scheduleResponse(key repairKey, packet Packet) { + jitter := time.Duration(0) + if s.cfg.RepairJitter > 0 { + span := int64(s.cfg.RepairJitter)*2 + 1 + jitter = time.Duration(s.rng.Int63n(span)) - s.cfg.RepairJitter + } + at := s.now + s.cfg.RepairLatency + jitter + if at < s.now { + at = s.now + } + if at < s.nextWireAt { + at = s.nextWireAt + } + if s.cfg.BandwidthBytesPerSec > 0 { + wire := time.Duration(float64(len(packet.Bytes)) / float64(s.cfg.BandwidthBytesPerSec) * float64(time.Second)) + if wire < time.Nanosecond { + wire = time.Nanosecond + } + at += wire + s.nextWireAt = at + } + d := delivery{at: at, sequence: s.sequence, packet: packet, key: key, primary: true, fromRepair: true} + s.sequence++ + if s.cfg.PacketLoss > 0 && s.rng.Float64() < s.cfg.PacketLoss { + d.drop = true + } + if s.corruptLeft > 0 { + d.corrupt = true + s.corruptLeft-- + } + heap.Push(&s.queue, d) + if !d.drop && s.cfg.DuplicateProbability > 0 && s.rng.Float64() < s.cfg.DuplicateProbability { + dup := d + dup.at++ + dup.sequence = s.sequence + dup.primary = false + dup.duplicate = true + dup.corrupt = false + s.sequence++ + heap.Push(&s.queue, dup) + } + if len(s.queue) > s.result.QueueHighWater { + s.result.QueueHighWater = len(s.queue) + } +} + +func (s *simulation) ingest(packet Packet, fromRepair, corrupt bool) error { + raw := packet.Bytes + if corrupt { + raw = append([]byte(nil), raw...) + if len(raw) > 200 { + raw[200] ^= 0x80 + } else if len(raw) > 0 { + raw[len(raw)-1] ^= 0x80 + } + } + started := time.Now() + shred, err := turbine.ParseShred(raw) + s.stageCPU["shred_parse"] += time.Since(started) + if err != nil { + return err + } + started = time.Now() + err = s.shredVerifier.Verify(shred, s.ledger.LeaderPub) + s.stageCPU["shred_validation"] += time.Since(started) + if err != nil { + return err + } + s.record("shred_validation", packet, "Merkle proof and leader signature valid") + if _, ok := s.firstShred[shred.Slot]; !ok { + s.firstShred[shred.Slot] = s.now + s.record("first_shred", packet, "") + } + started = time.Now() + spooled := s.spool.AppendShred(shred, raw) + s.stageCPU["blockstore_insert"] += time.Since(started) + if spooled { + s.record("blockstore_insert", packet, "verified shred spool") + } + beforeRecovered := s.assembler.RecoveredDataShreds() + started = time.Now() + blk, err := s.assembler.AddShredFrom(shred, fromRepair) + s.stageCPU["assembler_ingest_and_recovery"] += time.Since(started) + if err != nil { + return err + } + afterRecovered := s.assembler.RecoveredDataShreds() + if afterRecovered > beforeRecovered { + s.result.FECDecodes++ + if !s.haveRecoverAt { + s.firstRecoverAt = s.now + s.haveRecoverAt = true + } + s.record("fec_threshold_reached", packet, "observed at AddShredFrom call boundary") + s.record("fec_decode_complete", packet, fmt.Sprintf("recovered_data=%d", afterRecovered-beforeRecovered)) + } + if fromRepair { + s.record("repair_response_receive", packet, "") + } else { + s.record("live_shred_receive", packet, "") + } + if blk != nil { + canonical, ok := s.ledger.Slot(blk.Slot) + if !ok { + return fmt.Errorf("completed unknown slot %d", blk.Slot) + } + if err := compareBlock(canonical, blk); err != nil { + return err + } + if _, ok := s.spool.IsComplete(blk.Slot); !ok { + return fmt.Errorf("slot %d completed without spool completion record", blk.Slot) + } + s.completed[blk.Slot] = blk + for _, data := range canonical.Data { + s.result.DataShredBytesReplayable += uint64(len(data.Bytes)) + } + s.record("slot_offered_to_replay", packet, "verified block emitted") + s.advanceReplayable() + } + return nil +} + +func (s *simulation) advanceReplayable() { + for { + if _, ok := s.completed[s.nextReplaySlot]; !ok { + return + } + s.replayableAt[s.nextReplaySlot] = s.now + s.recordAt("slot_replayable", s.nextReplaySlot, 0, 0, 0, 0, "contiguous parent chain available") + s.nextReplaySlot++ + } +} + +func compareBlock(canonical *Slot, got *block.Block) error { + if got.Slot != canonical.Number || got.SourceParentSlot != canonical.ParentSlot { + return fmt.Errorf("block identity got slot=%d parent=%d, want slot=%d parent=%d", got.Slot, got.SourceParentSlot, canonical.Number, canonical.ParentSlot) + } + if len(got.Entries) != len(canonical.Entries) { + return fmt.Errorf("slot %d entries=%d, want %d", got.Slot, len(got.Entries), len(canonical.Entries)) + } + for i := range canonical.Entries { + want := canonical.Entries[i] + entry := got.Entries[i] + if entry.NumHashes != want.NumHashes || string(entry.Hash) != string(want.Hash[:]) || len(entry.Indices) != len(want.Txns) { + return fmt.Errorf("slot %d entry %d differs from canonical ledger", got.Slot, i) + } + } + if !got.TransactionSignaturesVerified() { + return fmt.Errorf("slot %d block was not signature-verified", got.Slot) + } + return nil +} + +func (s *simulation) record(stage string, packet Packet, detail string) { + s.recordAt(stage, packet.Slot, packet.FECSetIndex, packet.Index, packet.Type, len(packet.Bytes), detail) +} + +func (s *simulation) recordAt(stage string, slot uint64, fec, index uint32, typ turbine.ShredType, bytes int, detail string) { + if s.cfg.CollectTrace { + s.trace = append(s.trace, TraceEvent{ + Sequence: s.sequence, AtNanos: int64(s.now), Stage: stage, Slot: slot, + FECSetIndex: fec, ShredIndex: index, ShredType: typ, Bytes: bytes, Detail: detail, + }) + } + s.sequence++ +} + +func summarizeLatencies(values []time.Duration) LatencySummary { + if len(values) == 0 { + return LatencySummary{} + } + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + percentile := func(p float64) time.Duration { + index := int(float64(len(values)-1)*p + 0.5) + return values[index] + } + return LatencySummary{P50: percentile(.50), P95: percentile(.95), P99: percentile(.99)} +} + +func initialMissingDataShreds(ledger *Ledger, availability Availability) uint64 { + var heldPerFEC int + switch availability { + case AvailabilityComplete: + heldPerFEC = 32 + case AvailabilityNearLoss: + var missing uint64 + for i := range ledger.Slots { + for fec := range ledger.Slots[i].FECs { + if fec%2 == 0 { + missing += 2 + } else { + missing++ + } + } + } + return missing + case AvailabilitySparse: + heldPerFEC = 2 + case AvailabilityMixed: + heldPerFEC = 16 + } + return uint64(len(ledger.Slots) * ledger.Config.FECsPerSlot * (dataShredsPerFEC - heldPerFEC)) +} + +type repairKey struct { + slot uint64 + index uint32 +} + +type delivery struct { + at time.Duration + sequence int + packet Packet + key repairKey + primary bool + fromRepair bool + drop bool + duplicate bool + corrupt bool +} + +type deliveryHeap []delivery + +func (h deliveryHeap) Len() int { return len(h) } +func (h deliveryHeap) Less(i, j int) bool { + if h[i].at != h[j].at { + return h[i].at < h[j].at + } + return h[i].sequence < h[j].sequence +} +func (h deliveryHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *deliveryHeap) Push(x any) { *h = append(*h, x.(delivery)) } +func (h *deliveryHeap) Pop() any { + old := *h + last := old[len(old)-1] + *h = old[:len(old)-1] + return last +} diff --git a/pkg/turbine/repairsim/sim_test.go b/pkg/turbine/repairsim/sim_test.go new file mode 100644 index 000000000..fc161768f --- /dev/null +++ b/pkg/turbine/repairsim/sim_test.go @@ -0,0 +1,254 @@ +package repairsim + +import ( + "reflect" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/turbine" +) + +func testLedger(t *testing.T, slots, fecs int) *Ledger { + t.Helper() + ledger, err := GenerateLedger(LedgerConfig{ + StartSlot: 20_000, + Slots: slots, + FECsPerSlot: fecs, + Seed: 7, + ShredVersion: 11, + ReferenceTick: 63, + }) + if err != nil { + t.Fatal(err) + } + return ledger +} + +func deterministicConfig(scenario Scenario) Config { + cfg := DefaultConfig(scenario) + cfg.RepairLatency = 10 * time.Millisecond + cfg.RepairJitter = time.Millisecond + cfg.DuplicateProbability = 0 + cfg.BandwidthBytesPerSec = 0 + cfg.MaxConcurrent = 32 + cfg.Seed = 19 + return cfg +} + +func TestGenerateLedgerHasExactFECCountAndValidPackets(t *testing.T) { + ledger := testLedger(t, 2, 3) + if ledger.Config.EntriesPerSlot == 0 { + t.Fatal("entry count was not resolved") + } + for _, slot := range ledger.Slots { + if len(slot.FECs) != 3 { + t.Fatalf("slot %d FECs=%d, want 3", slot.Number, len(slot.FECs)) + } + for _, fec := range slot.FECs { + if len(fec.Data) != 32 || len(fec.Coding) != 32 { + t.Fatalf("slot %d FEC %d shape=%d+%d", slot.Number, fec.Index, len(fec.Data), len(fec.Coding)) + } + for _, packet := range append(append([]Packet(nil), fec.Data...), fec.Coding...) { + shred, err := parseAndVerify(packet, ledger) + if err != nil { + t.Fatalf("slot %d FEC %d: %v", slot.Number, fec.Index, err) + } + if shred.Slot != slot.Number || shred.FECSetIndex != fec.Index { + t.Fatalf("packet routing got slot=%d FEC=%d", shred.Slot, shred.FECSetIndex) + } + } + } + } +} + +func TestNearTipRepairCompletesAndTraceIsDeterministic(t *testing.T) { + ledger := testLedger(t, 3, 2) + cfg := deterministicConfig(ScenarioNearTip) + cfg.NaturalLateShreds = true + + first, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + second, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if first.CompletedSlots != 3 || first.SpoolCompleteSlots != 3 { + t.Fatalf("completed=%d spool=%d, want 3", first.CompletedSlots, first.SpoolCompleteSlots) + } + if first.LocallyRecoveredDataShreds == 0 || first.FECDecodes == 0 { + t.Fatalf("recovered=%d decodes=%d, want both nonzero", first.LocallyRecoveredDataShreds, first.FECDecodes) + } + if first.CanceledOrLateResponses == 0 { + t.Fatal("natural late-shred scenario did not produce canceled/late repair work") + } + if !reflect.DeepEqual(first.Trace, second.Trace) { + t.Fatal("same seed/config produced different logical traces") + } +} + +func TestNearTipWithoutRepairRemainsIncomplete(t *testing.T) { + ledger := testLedger(t, 2, 2) + cfg := deterministicConfig(ScenarioNearTip) + cfg.RepairEnabled = false + cfg.NaturalLateShreds = false + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != 0 || result.SpoolCompleteSlots != 0 { + t.Fatalf("completed=%d spool=%d without repair", result.CompletedSlots, result.SpoolCompleteSlots) + } +} + +func TestNaturalLateShredsWithoutRepair(t *testing.T) { + for _, tc := range []struct { + name string + fecs int + wantCompleted int + }{ + {name: "live-arrivals-complete-slots", fecs: 1, wantCompleted: 2}, + {name: "live-arrivals-leave-other-holes", fecs: 2, wantCompleted: 0}, + } { + t.Run(tc.name, func(t *testing.T) { + ledger := testLedger(t, 2, tc.fecs) + cfg := deterministicConfig(ScenarioNearTip) + cfg.RepairEnabled = false + cfg.NaturalLateShreds = true + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != tc.wantCompleted || result.SpoolCompleteSlots != tc.wantCompleted { + t.Fatalf("completed=%d spool=%d, want %d", result.CompletedSlots, result.SpoolCompleteSlots, tc.wantCompleted) + } + // Each live arrival provides the 32nd shard in its slot's first FEC, + // recovering one missing data shred without any repair response. + if result.LocallyRecoveredDataShreds != 2 { + t.Fatalf("recovered=%d, want 2 after both live arrivals", result.LocallyRecoveredDataShreds) + } + if result.RepairRequests != 0 || result.RepairResponses != 0 || result.RepairBytesRequested != 0 { + t.Fatalf("repair disabled: requests=%d responses=%d bytes requested=%d", result.RepairRequests, result.RepairResponses, result.RepairBytesRequested) + } + if result.LogicalElapsed < cfg.RepairLatency/2+time.Microsecond { + t.Fatalf("elapsed=%v, stopped before the last live arrival", result.LogicalElapsed) + } + }) + } +} + +func TestCompleteDeliveryEstablishesZeroRepairBaseline(t *testing.T) { + ledger := testLedger(t, 2, 2) + cfg := deterministicConfig(ScenarioNearTip) + cfg.Availability = AvailabilityComplete + cfg.RepairEnabled = false + cfg.NaturalLateShreds = false + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != 2 { + t.Fatalf("completed=%d, want 2", result.CompletedSlots) + } + if result.RepairRequests != 0 || result.LocallyRecoveredDataShreds != 0 { + t.Fatalf("baseline requests=%d recovered=%d, want zero", result.RepairRequests, result.LocallyRecoveredDataShreds) + } + if result.ShredEd25519Verifications != 4 || result.ShredSignatureCacheHits != 124 { + t.Fatalf("signature cache verifies=%d hits=%d, want 4/124 for four FEC roots", + result.ShredEd25519Verifications, result.ShredSignatureCacheHits) + } +} + +func TestDeepCatchupMixedUsesThresholdRecovery(t *testing.T) { + ledger := testLedger(t, 8, 2) + cfg := deterministicConfig(ScenarioDeepCatchup) + cfg.Availability = AvailabilityMixed + cfg.NaturalLateShreds = false + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != len(ledger.Slots) { + t.Fatalf("completed=%d, want %d", result.CompletedSlots, len(ledger.Slots)) + } + if result.LocallyRecoveredDataShreds <= result.UsefulNetworkDataShreds { + t.Fatalf("local recovery=%d, network data=%d; mixed threshold scenario should recover most losses locally", result.LocallyRecoveredDataShreds, result.UsefulNetworkDataShreds) + } + if result.RepairRequests == 0 || result.RepairBytesReceived == 0 { + t.Fatal("deep catch-up completed without exercising repair") + } +} + +func TestCorruptRepairResponseRejectedThenRetried(t *testing.T) { + ledger := testLedger(t, 1, 1) + cfg := deterministicConfig(ScenarioDeepCatchup) + cfg.Availability = AvailabilityMixed + cfg.CorruptResponses = 1 + result, err := Run(ledger, cfg) + if err != nil { + t.Fatal(err) + } + if result.CompletedSlots != 1 { + t.Fatalf("completed=%d, want 1", result.CompletedSlots) + } + if result.RejectedCorruptResponses != 1 { + t.Fatalf("rejected corrupt=%d, want 1", result.RejectedCorruptResponses) + } + if result.RepairRequests < 2 { + t.Fatalf("requests=%d, want retry after corruption", result.RepairRequests) + } +} + +func parseAndVerify(packet Packet, ledger *Ledger) (*turbine.Shred, error) { + shred, err := turbine.ParseShred(packet.Bytes) + if err != nil { + return nil, err + } + if err := shred.VerifySignature(ledger.LeaderPub); err != nil { + return nil, err + } + return shred, nil +} + +func BenchmarkScenarios(b *testing.B) { + ledger, err := GenerateLedger(LedgerConfig{ + StartSlot: 30_000, Slots: 8, FECsPerSlot: 2, Seed: 23, ShredVersion: 1, ReferenceTick: 63, + }) + if err != nil { + b.Fatal(err) + } + tests := []struct { + name string + scenario Scenario + availability Availability + }{ + {name: "near-tip", scenario: ScenarioNearTip, availability: AvailabilityNearLoss}, + {name: "deep-mixed", scenario: ScenarioDeepCatchup, availability: AvailabilityMixed}, + {name: "deep-sparse", scenario: ScenarioDeepCatchup, availability: AvailabilitySparse}, + } + for _, tt := range tests { + b.Run(tt.name, func(b *testing.B) { + cfg := DefaultConfig(tt.scenario) + cfg.Availability = tt.availability + cfg.RepairLatency = 0 + cfg.RepairJitter = 0 + cfg.DuplicateProbability = 0 + cfg.BandwidthBytesPerSec = 0 + cfg.NaturalLateShreds = false + cfg.CollectTrace = false + b.ReportAllocs() + for i := 0; i < b.N; i++ { + result, err := Run(ledger, cfg) + if err != nil { + b.Fatal(err) + } + if result.CompletedSlots != len(ledger.Slots) { + b.Fatalf("completed=%d", result.CompletedSlots) + } + b.ReportMetric(float64(result.RepairRequests), "repair-requests/op") + b.ReportMetric(float64(result.LocallyRecoveredDataShreds), "recovered-shreds/op") + } + }) + } +} diff --git a/pkg/turbine/shred.go b/pkg/turbine/shred.go index dae24ddd9..605faf814 100644 --- a/pkg/turbine/shred.go +++ b/pkg/turbine/shred.go @@ -410,13 +410,12 @@ func merkleHashNode(left []byte, right []byte) solana.Hash { return hashv([][]byte{[]byte(merkleHashPrefixNode), left, right}) } -func hashv(parts [][]byte) solana.Hash { +func hashv(parts [][]byte) (out solana.Hash) { h := sha256.New() for _, part := range parts { _, _ = h.Write(part) } - var out solana.Hash - copy(out[:], h.Sum(nil)) + _ = h.Sum(out[:0]) return out } diff --git a/pkg/turbine/sigcache.go b/pkg/turbine/sigcache.go index 7f3b13139..46358b07b 100644 --- a/pkg/turbine/sigcache.go +++ b/pkg/turbine/sigcache.go @@ -20,7 +20,13 @@ import ( // hit reproduces exactly the result of re-running it on the same inputs. // Tampered content can never hit — different bytes yield a different root, // hence a different key. Failures are never cached. -type shredSigCache struct { +// ShredSignatureVerifier authenticates Merkle shreds with the same bounded, +// per-root result cache used by UDPReceiver. The cache never stores failures; +// each packet's Merkle proof is still evaluated before a cache lookup. +// +// It is exported so deterministic and loopback ingress harnesses can exercise +// production validation without constructing a UDPReceiver. +type ShredSignatureVerifier struct { mu sync.Mutex cur map[shredSigCacheKey]struct{} prev map[shredSigCacheKey]struct{} @@ -29,6 +35,10 @@ type shredSigCache struct { verifies atomic.Uint64 } +// Keep the internal receiver/test name as an alias; there is one +// implementation and one cache contract. +type shredSigCache = ShredSignatureVerifier + type shredSigCacheKey struct { leader solana.PublicKey root solana.Hash @@ -42,7 +52,7 @@ const shredSigCacheGenCap = 4096 // verifyShred authenticates a shred exactly like Shred.VerifySignature, with // the per-root ed25519 result cached. -func (c *shredSigCache) verifyShred(s *Shred, leader solana.PublicKey) error { +func (c *ShredSignatureVerifier) verifyShred(s *Shred, leader solana.PublicKey) error { root, err := s.MerkleRoot() if err != nil { return err @@ -74,7 +84,13 @@ func (c *shredSigCache) verifyShred(s *Shred, leader solana.PublicKey) error { return nil } -func (c *shredSigCache) addLocked(key shredSigCacheKey) { +// Verify authenticates one shred and retains successful root/signature tuples +// for sibling shreds in the same FEC set. +func (c *ShredSignatureVerifier) Verify(s *Shred, leader solana.PublicKey) error { + return c.verifyShred(s, leader) +} + +func (c *ShredSignatureVerifier) addLocked(key shredSigCacheKey) { if c.cur == nil { c.cur = make(map[shredSigCacheKey]struct{}, shredSigCacheGenCap) } @@ -85,6 +101,11 @@ func (c *shredSigCache) addLocked(key shredSigCacheKey) { c.cur[key] = struct{}{} } -func (c *shredSigCache) stats() (hits, verifies uint64) { +func (c *ShredSignatureVerifier) stats() (hits, verifies uint64) { return c.hits.Load(), c.verifies.Load() } + +// Stats reports cache hits and actual Ed25519 verifications. +func (c *ShredSignatureVerifier) Stats() (hits, verifies uint64) { + return c.stats() +}