diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index c9c964064..19627f6cf 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -2565,7 +2565,16 @@ postBootstrap: if rpcPort < 0 || rpcPort > 65535 { klog.Fatalf("invalid port: %d", rpcPort) } else if rpcPort != 0 { - rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), epochScheduleFromState(mithrilState), solana.MustHashFromBase58(networkGenesisHash)) + schedule := epochScheduleFromState(mithrilState) + rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), schedule, solana.MustHashFromBase58(networkGenesisHash)) + if alpenglowMode { + if schedule == nil || schedule.SlotsPerEpoch == 0 || schedule.SlotsPerEpoch > math.MaxUint64/3 { + klog.Fatalf("Alpenglow RPC requires a valid epoch schedule") + } + if err := rpcServer.EnableEpochRewards(filepath.Join(accountsPath, "rpc-epoch-rewards"), 3*schedule.SlotsPerEpoch, mithrilState.LastRootedSlot); err != nil { + klog.Fatalf("unable to load RPC epoch rewards: %v", err) + } + } rpcServer.Start() mlog.Log.Infof("Started RPC server on port %d", rpcPort) } diff --git a/pkg/epochrewards/store.go b/pkg/epochrewards/store.go new file mode 100644 index 000000000..9397254f9 --- /dev/null +++ b/pkg/epochrewards/store.go @@ -0,0 +1,318 @@ +package epochrewards + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "sync/atomic" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go/rpc" +) + +const recordVersion = 1 + +// Record describes one account's inflation reward for an epoch. +type Record struct { + Epoch uint64 `json:"epoch"` + EffectiveSlot uint64 `json:"effectiveSlot"` + Address string `json:"address"` + Amount uint64 `json:"amount"` + PostBalance uint64 `json:"postBalance"` + Commission *uint8 `json:"commission,omitempty"` +} + +type rewardKey struct { + epoch uint64 + address string +} + +type batch struct { + Version uint32 `json:"version"` + Through uint64 `json:"through"` + Records []Record `json:"records"` +} + +// Store keeps speculative rewards in memory and persists them with the same +// rooted fold that commits the credited account balances. +type Store struct { + dir string + retentionSlots uint64 + rooted atomic.Uint64 + mu sync.RWMutex + pending map[rewardKey]Record + persisted map[rewardKey]Record + sourceBatch map[rewardKey]uint64 + batches map[uint64][]rewardKey +} + +// Open loads or creates an epoch reward store. +func Open(dir string, retentionSlots uint64) (*Store, error) { + if retentionSlots == 0 { + return nil, errors.New("epoch reward retention must be greater than zero") + } + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, fmt.Errorf("create epoch reward directory: %w", err) + } + store := &Store{ + dir: dir, + retentionSlots: retentionSlots, + pending: make(map[rewardKey]Record), + persisted: make(map[rewardKey]Record), + sourceBatch: make(map[rewardKey]uint64), + batches: make(map[uint64][]rewardKey), + } + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read epoch reward directory: %w", err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasPrefix(entry.Name(), "batch-") || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + through, err := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(entry.Name(), "batch-"), ".json"), 10, 64) + if err != nil { + continue + } + loaded, err := readBatch(filepath.Join(dir, entry.Name())) + if err != nil { + return nil, fmt.Errorf("load epoch reward batch %d: %w", through, err) + } + if loaded.Through != through { + return nil, fmt.Errorf("load epoch reward batch %d: record contains through %d", through, loaded.Through) + } + store.install(loaded) + } + return store, nil +} + +// RecordBlock adds inflation rewards from a successfully replayed block. +func (s *Store) RecordBlock(block *b.Block) error { + if block == nil { + return errors.New("record epoch rewards: nil block") + } + if block.Epoch == 0 { + return nil + } + rewardedEpoch := block.Epoch - 1 + s.mu.Lock() + defer s.mu.Unlock() + for _, reward := range block.Rewards { + if reward.Lamports < 0 || (reward.RewardType != rpc.RewardTypeVoting && reward.RewardType != rpc.RewardTypeStaking) { + continue + } + address := reward.Pubkey.String() + commission := cloneUint8(reward.Commission) + record := Record{ + Epoch: rewardedEpoch, EffectiveSlot: block.Slot, Address: address, + Amount: uint64(reward.Lamports), PostBalance: reward.PostBalance, Commission: commission, + } + s.pending[rewardKey{epoch: rewardedEpoch, address: address}] = record + } + return nil +} + +// Prepare writes rewards through a fold boundary before the fold is committed. +func (s *Store) Prepare(through uint64) error { + s.mu.RLock() + records := make([]Record, 0) + for _, record := range s.pending { + if record.EffectiveSlot <= through { + records = append(records, record) + } + } + s.mu.RUnlock() + if len(records) == 0 { + return nil + } + sort.Slice(records, func(i, j int) bool { + if records[i].Epoch != records[j].Epoch { + return records[i].Epoch < records[j].Epoch + } + return records[i].Address < records[j].Address + }) + payload := batch{Version: recordVersion, Through: through, Records: records} + tmp, err := os.CreateTemp(s.dir, ".epoch-rewards-*") + if err != nil { + return fmt.Errorf("create epoch reward temp file: %w", err) + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + writeErr := json.NewEncoder(tmp).Encode(payload) + if writeErr == nil { + writeErr = tmp.Sync() + } + closeErr := tmp.Close() + if writeErr != nil { + return fmt.Errorf("write epoch rewards through slot %d: %w", through, writeErr) + } + if closeErr != nil { + return fmt.Errorf("close epoch rewards through slot %d: %w", through, closeErr) + } + if err := os.Rename(tmpPath, s.batchPath(through)); err != nil { + return fmt.Errorf("publish epoch rewards through slot %d: %w", through, err) + } + if err := syncDir(s.dir); err != nil { + return err + } + s.mu.Lock() + s.install(payload) + s.mu.Unlock() + return nil +} + +func (s *Store) install(value batch) { + keys := make([]rewardKey, 0, len(value.Records)) + for _, record := range value.Records { + key := rewardKey{epoch: record.Epoch, address: record.Address} + s.persisted[key] = record + s.sourceBatch[key] = value.Through + keys = append(keys, key) + } + s.batches[value.Through] = keys +} + +// SetRooted advances the durable reward view and prunes stale batches. +func (s *Store) SetRooted(slot uint64) error { + s.mu.Lock() + for key, record := range s.pending { + if record.EffectiveSlot <= slot { + delete(s.pending, key) + } + } + pruneBefore := uint64(0) + if slot > s.retentionSlots { + pruneBefore = slot - s.retentionSlots + } + removed := false + for through, keys := range s.batches { + orphaned := through > slot + expired := pruneBefore > 0 && through <= pruneBefore + if !orphaned && !expired { + continue + } + if err := os.Remove(s.batchPath(through)); err != nil && !os.IsNotExist(err) { + s.mu.Unlock() + return fmt.Errorf("prune epoch reward batch %d: %w", through, err) + } + removed = true + for _, key := range keys { + if s.sourceBatch[key] == through { + delete(s.persisted, key) + delete(s.sourceBatch, key) + } + } + delete(s.batches, through) + } + s.mu.Unlock() + if removed { + if err := syncDir(s.dir); err != nil { + return err + } + } + s.rooted.Store(slot) + return nil +} + +// Rewind removes speculative rewards produced by a discarded fork suffix. +func (s *Store) Rewind(fromSlot uint64) error { + s.mu.Lock() + for key, record := range s.pending { + if record.EffectiveSlot >= fromSlot { + delete(s.pending, key) + } + } + removed := false + for through, keys := range s.batches { + if through < fromSlot || through <= s.rooted.Load() { + continue + } + if err := os.Remove(s.batchPath(through)); err != nil && !os.IsNotExist(err) { + s.mu.Unlock() + return fmt.Errorf("rewind epoch reward batch %d: %w", through, err) + } + removed = true + for _, key := range keys { + if s.sourceBatch[key] == through { + delete(s.persisted, key) + delete(s.sourceBatch, key) + } + } + delete(s.batches, through) + } + s.mu.Unlock() + if removed { + return syncDir(s.dir) + } + return nil +} + +// Get returns an address's reward for an epoch. +func (s *Store) Get(epoch uint64, address string, includePending bool) (Record, bool) { + key := rewardKey{epoch: epoch, address: address} + s.mu.RLock() + defer s.mu.RUnlock() + if includePending { + if record, ok := s.pending[key]; ok { + return record, true + } + } + record, ok := s.persisted[key] + if !ok || record.EffectiveSlot > s.rooted.Load() || s.sourceBatch[key] > s.rooted.Load() { + return Record{}, false + } + return record, true +} + +// RootedSlot returns the durable reward watermark. +func (s *Store) RootedSlot() uint64 { + return s.rooted.Load() +} + +func (s *Store) batchPath(through uint64) string { + return filepath.Join(s.dir, fmt.Sprintf("batch-%020d.json", through)) +} + +func readBatch(path string) (batch, error) { + data, err := os.ReadFile(path) + if err != nil { + return batch{}, err + } + var value batch + if err := json.Unmarshal(data, &value); err != nil { + return batch{}, err + } + if value.Version != recordVersion { + return batch{}, fmt.Errorf("unsupported epoch reward version %d", value.Version) + } + return value, nil +} + +func syncDir(path string) error { + dir, err := os.Open(path) + if err != nil { + return fmt.Errorf("open epoch reward directory: %w", err) + } + if err := dir.Sync(); err != nil { + _ = dir.Close() + return fmt.Errorf("sync epoch reward directory: %w", err) + } + if err := dir.Close(); err != nil { + return fmt.Errorf("close epoch reward directory: %w", err) + } + return nil +} + +func cloneUint8(value *uint8) *uint8 { + if value == nil { + return nil + } + copy := *value + return © +} diff --git a/pkg/epochrewards/store_test.go b/pkg/epochrewards/store_test.go new file mode 100644 index 000000000..7ec496fbd --- /dev/null +++ b/pkg/epochrewards/store_test.go @@ -0,0 +1,170 @@ +package epochrewards + +import ( + "os" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestStoreKeepsConfirmedAndFinalizedViewsSeparate(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir, 256) + require.NoError(t, err) + + address := solana.NewWallet().PublicKey() + commission := uint8(7) + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 130, Epoch: 2, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 42, PostBalance: 1_042, + RewardType: rpc.RewardTypeVoting, Commission: &commission, + }}, + })) + + confirmed, ok := store.Get(1, address.String(), true) + require.True(t, ok) + require.Equal(t, uint64(42), confirmed.Amount) + require.Equal(t, uint64(130), confirmed.EffectiveSlot) + require.Equal(t, uint64(1_042), confirmed.PostBalance) + require.Equal(t, uint8(7), *confirmed.Commission) + _, ok = store.Get(1, address.String(), false) + require.False(t, ok) + + require.NoError(t, store.Prepare(130)) + require.NoError(t, store.SetRooted(130)) + finalized, ok := store.Get(1, address.String(), false) + require.True(t, ok) + require.Equal(t, confirmed, finalized) + + reopened, err := Open(dir, 256) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(130)) + restarted, ok := reopened.Get(1, address.String(), false) + require.True(t, ok) + require.Equal(t, finalized, restarted) +} + +func TestStoreHidesPreparedRewardsUntilTheirFoldCommits(t *testing.T) { + store, err := Open(t.TempDir(), 100) + require.NoError(t, err) + address := solana.PublicKey{1} + require.NoError(t, store.RecordBlock(&b.Block{Slot: 11, Epoch: 2, Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 5, RewardType: rpc.RewardTypeStaking, + }}})) + require.NoError(t, store.Prepare(12)) + store.rooted.Store(11) // The account fold through slot 12 has not committed. + _, ok := store.Get(1, address.String(), false) + require.False(t, ok) + require.NoError(t, store.SetRooted(12)) + _, ok = store.Get(1, address.String(), false) + require.True(t, ok) +} + +func TestStoreKeepsZeroInflationRewards(t *testing.T) { + store, err := Open(t.TempDir(), 32) + require.NoError(t, err) + address := solana.NewWallet().PublicKey() + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 33, Epoch: 1, + Rewards: []rpc.BlockReward{ + {Pubkey: address, Lamports: 10, RewardType: rpc.RewardTypeFee}, + {Pubkey: address, Lamports: 0, RewardType: rpc.RewardTypeStaking}, + }, + })) + reward, ok := store.Get(0, address.String(), true) + require.True(t, ok) + require.Zero(t, reward.Amount) + require.Equal(t, uint64(33), reward.EffectiveSlot) + require.NoError(t, store.Prepare(33)) + entries, err := batchFiles(store.dir) + require.NoError(t, err) + require.Len(t, entries, 1) +} + +func TestStoreRecordsLiveAlpenglowVotingRewardAfterVATDebit(t *testing.T) { + store, err := Open(t.TempDir(), 108_000) + require.NoError(t, err) + vote := solana.MustPublicKeyFromBase58("TitanB6gCvNeb5RLM1RuGNT1mgmqxR6Q8hXMALQpLuJ") + // A VAT debit must not hide the voting inflation credit in the same block. + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 7_236_004, Epoch: 134, + Rewards: []rpc.BlockReward{ + {Pubkey: vote, Lamports: -800_000_000, PostBalance: 19_022_056_333_577, RewardType: rpc.RewardType("VATDebit")}, + {Pubkey: vote, Lamports: 413_137_680_751, PostBalance: 19_435_194_014_328, RewardType: rpc.RewardTypeVoting}, + }, + })) + require.NoError(t, store.Prepare(7_236_004)) + require.NoError(t, store.SetRooted(7_236_004)) + reward, ok := store.Get(133, vote.String(), false) + require.True(t, ok) + require.Equal(t, uint64(7_236_004), reward.EffectiveSlot) + require.Equal(t, uint64(413_137_680_751), reward.Amount) + require.Equal(t, uint64(19_435_194_014_328), reward.PostBalance) + require.Nil(t, reward.Commission) +} + +func TestStoreDropsUnselectedAndExpiredBatches(t *testing.T) { + dir := t.TempDir() + store, err := Open(dir, 10) + require.NoError(t, err) + address := solana.NewWallet().PublicKey() + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 20, Epoch: 2, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 5, RewardType: rpc.RewardTypeStaking}}, + })) + require.NoError(t, store.Prepare(20)) + + reopened, err := Open(dir, 10) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(19)) + _, ok := reopened.Get(1, address.String(), false) + require.False(t, ok, "a batch ahead of the recovered root is an orphan") + + require.NoError(t, reopened.RecordBlock(&b.Block{ + Slot: 21, Epoch: 2, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 6, RewardType: rpc.RewardTypeStaking}}, + })) + require.NoError(t, reopened.Prepare(21)) + require.NoError(t, reopened.SetRooted(21)) + _, ok = reopened.Get(1, address.String(), false) + require.True(t, ok) + require.NoError(t, reopened.SetRooted(32)) + _, ok = reopened.Get(1, address.String(), false) + require.False(t, ok, "records outside retention are removed") +} + +func TestStoreRewindsSpeculativeRewards(t *testing.T) { + store, err := Open(t.TempDir(), 32) + require.NoError(t, err) + address := solana.NewWallet().PublicKey() + require.NoError(t, store.RecordBlock(&b.Block{ + Slot: 33, Epoch: 1, + Rewards: []rpc.BlockReward{{Pubkey: address, Lamports: 5, RewardType: rpc.RewardTypeVoting}}, + })) + require.NoError(t, store.Prepare(33)) + + require.NoError(t, store.Rewind(33)) + _, ok := store.Get(0, address.String(), true) + require.False(t, ok) + files, err := batchFiles(store.dir) + require.NoError(t, err) + require.Empty(t, files) +} + +func batchFiles(dir string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + files := make([]string, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, entry.Name()) + } + } + return files, nil +} diff --git a/pkg/replay/block.go b/pkg/replay/block.go index e571c98d0..52de989ea 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -52,6 +52,13 @@ type SlotCtxSetter interface { SetSlotCtx(slotCtx *sealevel.SlotCtx) } +type epochRewardPublisher interface { + RecordEpochRewards(block *b.Block) error + PrepareEpochRewards(through uint64) error + SetRootedEpochRewardsSlot(slot uint64) error + RewindEpochRewards(fromSlot uint64) error +} + // BlockFetchOpts contains options for parallel block fetching type BlockFetchOpts struct { MaxRPS int // Rate limit (requests per second), 0 = use default @@ -1689,6 +1696,13 @@ func ReplayBlocks( ) *ReplayResult { result := &ReplayResult{} alpenglowMode := consensusOpts != nil && consensusOpts.Alpenglow + rewardPublisher, _ := rpcServer.(epochRewardPublisher) + if rewardPublisher != nil { + if err := rewardPublisher.RewindEpochRewards(startSlot); err != nil { + result.Error = fmt.Errorf("rewind epoch rewards from replay slot %d: %w", startSlot, err) + return result + } + } replayFrontier := uint64(0) if startSlot > 0 { replayFrontier = startSlot - 1 @@ -2006,7 +2020,16 @@ func ReplayBlocks( // on the existing fold worker, after releasing the live cache lock. Capture: transactionStatuses.CaptureSnapshotThrough, Install: func(through uint64, payload []byte) (*state.TransactionStatusCheckpointRef, error) { - return PrepareTransactionStatusCheckpoint(acctsDbPath, through, payload) + ref, err := PrepareTransactionStatusCheckpoint(acctsDbPath, through, payload) + if err != nil { + return nil, err + } + if rewardPublisher != nil { + if err := rewardPublisher.PrepareEpochRewards(through); err != nil { + return nil, fmt.Errorf("prepare epoch rewards: %w", err) + } + } + return ref, nil }, AfterCommit: checkpointAfterCommit, }); hookErr != nil { @@ -2076,6 +2099,11 @@ func ReplayBlocks( mithrilState.LastRootedSlot = promotedThrough mithrilState.LastRootedBankhash = rootedCtx.Bankhash mithrilState.LastRootedContext = rootedCtx + if rewardPublisher != nil { + if err := rewardPublisher.SetRootedEpochRewardsSlot(promotedThrough); err != nil { + mlog.Log.Errorf("failed to advance rooted epoch rewards through slot %d: %v", promotedThrough, err) + } + } if rewardsCompletion.retire(&partitionedRewardsInfo, promotedThrough) { rewardsHoldBelowSlot = 0 mlog.Log.Infof("epoch rewards bookkeeping retired through durable slot %d; later fork switches may unwind in memory", promotedThrough) @@ -2503,6 +2531,13 @@ func ReplayBlocks( mlog.Log.Warnf("%v", result.Error) return false } + if rewardPublisher != nil { + if err := rewardPublisher.RewindEpochRewards(sw.Slot); err != nil { + result.Error = fmt.Errorf("%w: rewind epoch rewards: %v", sw, err) + mlog.Log.Warnf("%v", result.Error) + return false + } + } for slot := range alpenglowExecutedBlockIDs { if slot >= sw.Slot { @@ -3108,6 +3143,11 @@ func ReplayBlocks( // post-epoch boundary rewards distribution if partitionedEpochRewardsEnabled && partitionedRewardsInfo != nil && currentSlot >= partitionedRewardsInfo.FirstStakingRewardSlot && partitionedRewardsInfo.NumRewardPartitionsRemaining > 0 { distributedAccts, parentDistributedAccts := distributePartitionedEpochRewardsForSlot(acctsDb, lastSlotCtx, block.EpochUpdatedAccts, replayCtx, partitionedRewardsInfo, currentSlot, block.BlockHeight) + if err := replaceInflationRewardRecords(block, distributedAccts, parentDistributedAccts, rpc.RewardTypeStaking); err != nil { + result.Error = err + mlog.Log.Errorf("%v", err) + break + } block.EpochUpdatedAccts = append(block.EpochUpdatedAccts, distributedAccts...) block.ParentEpochUpdatedAccts = append(block.ParentEpochUpdatedAccts, parentDistributedAccts...) } @@ -3155,6 +3195,12 @@ func ReplayBlocks( global.ClearPendingStakePubkeys() break } + if rewardPublisher != nil { + if err := rewardPublisher.RecordEpochRewards(block); err != nil { + result.Error = fmt.Errorf("record epoch rewards for slot %d: %w", block.Slot, err) + break + } + } // The successful child now owns its derived snapshot. Any later bank uses // lastSlotCtx; the one-shot retained unwind bridge is no longer needed. rewardsCompletion.observeBank(partitionedRewardsInfo, lastSlotCtx.BankSysvars()) diff --git a/pkg/replay/inflation_reward_records_test.go b/pkg/replay/inflation_reward_records_test.go new file mode 100644 index 000000000..909e470dd --- /dev/null +++ b/pkg/replay/inflation_reward_records_test.go @@ -0,0 +1,143 @@ +package replay + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" + "github.com/Overclock-Validator/mithril/pkg/rewards" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestInflationRewardsSurviveCompletionFoldRetirementAndRestart(t *testing.T) { + dir := t.TempDir() + store, err := epochrewards.Open(dir, 100) + require.NoError(t, err) + require.NoError(t, store.SetRooted(4)) + key := solana.PublicKey{1} + require.NoError(t, store.RecordBlock(&b.Block{Slot: 5, Epoch: 1, Rewards: []rpc.BlockReward{ + {Pubkey: key, Lamports: 7, PostBalance: 107, RewardType: rpc.RewardTypeVoting}, + }})) + committer := &fakeCommitter{durable: accounts.NewMemAccounts(), failOn: 7} + tail := asyncTestTail(committer, 5, 6, 7, 8) + statuses := NewTransactionStatusCache() + checkpointDir := t.TempDir() + require.NoError(t, tail.SetTransactionStatusCheckpointHooks(TransactionStatusCheckpointHooks{ + Capture: statuses.CaptureSnapshotThrough, + Install: func(through uint64, payload []byte) (*state.TransactionStatusCheckpointRef, error) { + ref, err := PrepareTransactionStatusCheckpoint(checkpointDir, through, payload) + if err != nil { + return nil, err + } + return ref, store.Prepare(through) + }, + })) + info := &rewards.PartitionedRewardDistributionInfo{} + var completion partitionedRewardsCompletion + completion.observeBank(info, testUnwindBankSysvars(t, 7, 50)) + job, err := tail.buildRewardsCompletionFoldJob(completion.slot) + require.NoError(t, err) + require.Error(t, runFoldJob(committer, job)) + _, visible := store.Get(0, key.String(), false) + require.False(t, visible, "a prepared reward must not precede its committed bank") + require.False(t, completion.retire(&info, store.RootedSlot())) + + committer.failOn = 0 + job, err = tail.buildRewardsCompletionFoldJob(completion.slot) + require.NoError(t, err) + require.NoError(t, runFoldJob(committer, job)) + require.NotNil(t, tail.applyFoldJob(job)) + require.NoError(t, store.SetRooted(job.through)) + require.True(t, completion.retire(&info, job.through)) + want, visible := store.Get(0, key.String(), false) + require.True(t, visible) + require.Equal(t, uint64(7), want.Amount) + + reopened, err := epochrewards.Open(dir, 100) + require.NoError(t, err) + require.NoError(t, reopened.SetRooted(job.through)) + got, visible := reopened.Get(0, key.String(), false) + require.True(t, visible) + require.Equal(t, want, got) +} + +func TestReplaceInflationRewardRecordsUsesAppliedBalanceDelta(t *testing.T) { + key := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12, Rewards: []rpc.BlockReward{ + {Pubkey: key, Lamports: 9, RewardType: rpc.RewardTypeVoting}, + {Pubkey: key, Lamports: 2, RewardType: rpc.RewardTypeFee}, + }} + parent := &accounts.Account{Key: key, Lamports: 100} + updated := &accounts.Account{Key: key, Lamports: 107} + + require.NoError(t, replaceInflationRewardRecords( + block, + []*accounts.Account{updated, nil}, + []*accounts.Account{parent, nil}, + rpc.RewardTypeVoting, + )) + require.Len(t, block.Rewards, 2) + require.Equal(t, rpc.RewardTypeFee, block.Rewards[0].RewardType) + require.Equal(t, rpc.RewardTypeVoting, block.Rewards[1].RewardType) + require.Equal(t, int64(7), block.Rewards[1].Lamports) + require.Equal(t, uint64(107), block.Rewards[1].PostBalance) +} + +func TestReplaceInflationRewardRecordsRejectsInvalidAccountPairs(t *testing.T) { + key := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12} + require.Error(t, replaceInflationRewardRecords( + block, + []*accounts.Account{{Key: key, Lamports: 99}}, + []*accounts.Account{{Key: key, Lamports: 100}}, + rpc.RewardTypeStaking, + )) + require.Error(t, replaceInflationRewardRecords( + block, + []*accounts.Account{{Key: key, Lamports: 101}}, + nil, + rpc.RewardTypeStaking, + )) +} + +func TestReplaceInflationRewardRecordsKeepsZeroAmount(t *testing.T) { + key := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12} + parent := &accounts.Account{Key: key, Lamports: 100} + updated := &accounts.Account{Key: key, Lamports: 100} + + require.NoError(t, replaceInflationRewardRecords( + block, + []*accounts.Account{updated}, + []*accounts.Account{parent}, + rpc.RewardTypeVoting, + )) + require.Len(t, block.Rewards, 1) + require.Zero(t, block.Rewards[0].Lamports) + require.Equal(t, uint64(100), block.Rewards[0].PostBalance) +} + +func TestReplaceInflationRewardRecordsOmitsEpochRewardsSysvar(t *testing.T) { + stake := solana.NewWallet().PublicKey() + block := &b.Block{Slot: 12} + require.NoError(t, replaceInflationRewardRecords( + block, + []*accounts.Account{ + {Key: stake, Lamports: 105}, + {Key: sealevel.SysvarEpochRewardsAddr, Lamports: 1}, + }, + []*accounts.Account{ + {Key: stake, Lamports: 100}, + {Key: sealevel.SysvarEpochRewardsAddr, Lamports: 1}, + }, + rpc.RewardTypeStaking, + )) + require.Len(t, block.Rewards, 1) + require.Equal(t, stake, block.Rewards[0].Pubkey) + require.Equal(t, int64(5), block.Rewards[0].Lamports) +} diff --git a/pkg/replay/rewards.go b/pkg/replay/rewards.go index 8f92a9bfd..3668d38ff 100644 --- a/pkg/replay/rewards.go +++ b/pkg/replay/rewards.go @@ -4,6 +4,7 @@ import ( "bytes" "encoding/json" "fmt" + "math" "os" "path/filepath" "sort" @@ -493,6 +494,9 @@ func beginPartitionedEpochRewardsDistribution(acctsDb *accountsdb.AccountsDb, sl rewardLoader := epochRewardAccountLoader(acctsDb, slot, slotCtx, stagedEpochAccts) updatedAccts, parentUpdatedAccts, voteRewardsDistributed := rewards.DistributeVotingRewards(acctsDb, streamResult.ValidatorRewards, slot, rewardLoader) + if err := replaceInflationRewardRecords(block, updatedAccts, parentUpdatedAccts, rpc.RewardTypeVoting); err != nil { + panic(err) + } newEpochRewards := sealevel.SysvarEpochRewards{DistributionStartingBlockHeight: block.BlockHeight + 1, NumPartitions: streamResult.NumPartitions, ParentBlockhash: block.LastBlockhash, @@ -585,3 +589,40 @@ func distributePartitionedEpochRewardsForSlot(acctsDb *accountsdb.AccountsDb, pa return distributedAccts, parentDistributedAccts } + +func replaceInflationRewardRecords(block *block.Block, updated, parents []*accounts.Account, rewardType rpc.RewardType) error { + if block == nil { + return fmt.Errorf("record %s rewards: nil block", rewardType) + } + if len(updated) != len(parents) { + return fmt.Errorf("record %s rewards at slot %d: %d updated accounts, %d parents", rewardType, block.Slot, len(updated), len(parents)) + } + records := make([]rpc.BlockReward, 0, len(updated)) + for index, account := range updated { + if account == nil || account.Key == sealevel.SysvarEpochRewardsAddr { + continue + } + parent := parents[index] + if parent == nil { + return fmt.Errorf("record %s rewards at slot %d: account %s has no parent", rewardType, block.Slot, account.Key) + } + if account.Lamports < parent.Lamports { + return fmt.Errorf("record %s rewards at slot %d: account %s balance decreased from %d to %d", rewardType, block.Slot, account.Key, parent.Lamports, account.Lamports) + } + amount := account.Lamports - parent.Lamports + if amount > math.MaxInt64 { + return fmt.Errorf("record %s rewards at slot %d: account %s reward %d overflows int64", rewardType, block.Slot, account.Key, amount) + } + records = append(records, rpc.BlockReward{ + Pubkey: account.Key, Lamports: int64(amount), PostBalance: account.Lamports, RewardType: rewardType, + }) + } + kept := block.Rewards[:0] + for _, reward := range block.Rewards { + if reward.RewardType != rewardType { + kept = append(kept, reward) + } + } + block.Rewards = append(kept, records...) + return nil +} diff --git a/pkg/rewards/alpenglow_rewards_test.go b/pkg/rewards/alpenglow_rewards_test.go index ae9d451bb..61ba328d1 100644 --- a/pkg/rewards/alpenglow_rewards_test.go +++ b/pkg/rewards/alpenglow_rewards_test.go @@ -2,6 +2,7 @@ package rewards import ( "math" + "sync/atomic" "testing" "github.com/Overclock-Validator/mithril/pkg/features" @@ -88,6 +89,21 @@ func TestAlpenglowCommissionSplitPreservesFractionalLamport(t *testing.T) { }, tower) } +func TestZeroCommissionRetainsVotingRewardEntry(t *testing.T) { + votePubkey := solana.NewWallet().PublicKey() + voteState := &sealevel.VoteStateVersions{ + Type: sealevel.VoteStateVersionV4, + V4: sealevel.VoteState4{InflationRewardsCommissionBps: 0}, + } + split := voteCommissionSplit(voteState, 1_000, true, true) + require.Zero(t, split.VoterPortion) + + rewards := make(map[solana.PublicKey]*atomic.Uint64) + accumulateVotingReward(rewards, votePubkey, voteState, false, split.VoterPortion) + require.Contains(t, rewards, votePubkey) + require.Zero(t, rewards[votePubkey].Load()) +} + func TestV4CommissionKeepsBasisPointPrecision(t *testing.T) { voteState := &sealevel.VoteStateVersions{ Type: sealevel.VoteStateVersionV4, diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go index bfff0ebfc..8cb0ea963 100644 --- a/pkg/rewards/rewards.go +++ b/pkg/rewards/rewards.go @@ -644,6 +644,17 @@ func voteInflationRewardsCollector(votePubkey solana.PublicKey, voteState *seale return votePubkey } +func accumulateVotingReward(validatorRewards map[solana.PublicKey]*atomic.Uint64, votePubkey solana.PublicKey, voteState *sealevel.VoteStateVersions, customCollector bool, amount uint64) { + // Zero commission still produces a vote reward record. + collector := voteInflationRewardsCollector(votePubkey, voteState, customCollector) + accumulator := validatorRewards[collector] + if accumulator == nil { + accumulator = &atomic.Uint64{} + validatorRewards[collector] = accumulator + } + accumulator.Add(amount) +} + func calculateStakePointsAndCredits( pubkey solana.PublicKey, stakeHistory *sealevel.SysvarStakeHistory, @@ -1059,17 +1070,8 @@ func CalculateRewardsStreaming( return nil, fmt.Errorf("temp spool write failed: %w", err) } - if splitResult.VoterPortion > 0 { - collector := voteInflationRewardsCollector( - rec.VotePubkey, voteState, f.IsActive(features.CustomCommissionCollector), - ) - accumulator := validatorRewards[collector] - if accumulator == nil { - accumulator = &atomic.Uint64{} - validatorRewards[collector] = accumulator - } - accumulator.Add(splitResult.VoterPortion) - } + accumulateVotingReward(validatorRewards, rec.VotePubkey, voteState, + f.IsActive(features.CustomCommissionCollector), splitResult.VoterPortion) } pointsReader.Close() diff --git a/pkg/rpcserver/get_inflation_reward.go b/pkg/rpcserver/get_inflation_reward.go new file mode 100644 index 000000000..5f4e985f8 --- /dev/null +++ b/pkg/rpcserver/get_inflation_reward.go @@ -0,0 +1,92 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +const maxInflationRewardAddresses = 5 + +type inflationRewardConfig struct { + Commitment string `json:"commitment"` + Epoch *uint64 `json:"epoch"` + MinContextSlot *uint64 `json:"minContextSlot"` +} + +// InflationRewardResp is one getInflationReward result entry. +type InflationRewardResp struct { + Epoch uint64 `json:"epoch"` + EffectiveSlot uint64 `json:"effectiveSlot"` + Amount uint64 `json:"amount"` + PostBalance uint64 `json:"postBalance"` + Commission *uint8 `json:"commission"` +} + +func (rpcServer *RpcServer) GetInflationReward(ctx context.Context, p jsonrpc.RawParams) ([]*InflationRewardResp, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + var params []json.RawMessage + if err := json.Unmarshal(p, ¶ms); err != nil || len(params) < 1 || len(params) > 2 { + return nil, &InvalidParamsError{Message: "getInflationReward requires addresses and optional config"} + } + var addresses []string + if err := json.Unmarshal(params[0], &addresses); err != nil || addresses == nil { + return nil, &InvalidParamsError{Message: "invalid getInflationReward addresses"} + } + if len(addresses) > maxInflationRewardAddresses { + return nil, &InvalidParamsError{Message: fmt.Sprintf("Too many inputs provided; max %d", maxInflationRewardAddresses)} + } + for _, address := range addresses { + if _, err := solana.PublicKeyFromBase58(address); err != nil { + return nil, &InvalidParamsError{Message: fmt.Sprintf("invalid address %q", address)} + } + } + + config := inflationRewardConfig{Commitment: "finalized"} + if len(params) == 2 && string(params[1]) != "null" { + if err := json.Unmarshal(params[1], &config); err != nil { + return nil, &InvalidParamsError{Message: "invalid getInflationReward config"} + } + if config.Commitment == "" { + config.Commitment = "finalized" + } + } + if config.Commitment != "confirmed" && config.Commitment != "finalized" { + return nil, &InvalidParamsError{Message: "getInflationReward requires confirmed or finalized commitment"} + } + if rpcServer.epochRewards == nil { + return nil, fmt.Errorf("node has no retained epoch reward history") + } + if rpcServer.epochSchedule == nil { + return nil, fmt.Errorf("node has no epoch schedule available") + } + + contextSlot := rpcServer.epochRewards.RootedSlot() + if config.MinContextSlot != nil && contextSlot < *config.MinContextSlot { + return nil, &MinContextSlotNotReachedError{ContextSlot: contextSlot} + } + epoch := rpcServer.epochSchedule.GetEpoch(contextSlot) + if config.Epoch != nil { + epoch = *config.Epoch + } else if epoch > 0 { + epoch-- + } + + result := make([]*InflationRewardResp, len(addresses)) + for index, address := range addresses { + record, ok := rpcServer.epochRewards.Get(epoch, address, false) + if !ok { + continue + } + result[index] = &InflationRewardResp{ + Epoch: record.Epoch, EffectiveSlot: record.EffectiveSlot, + Amount: record.Amount, PostBalance: record.PostBalance, Commission: record.Commission, + } + } + return result, nil +} diff --git a/pkg/rpcserver/get_inflation_reward_test.go b/pkg/rpcserver/get_inflation_reward_test.go new file mode 100644 index 000000000..5edb2afce --- /dev/null +++ b/pkg/rpcserver/get_inflation_reward_test.go @@ -0,0 +1,143 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "testing" + + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/rpc" + "github.com/stretchr/testify/require" +) + +func TestGetInflationRewardCommitmentAndDefaults(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{ + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, + slotCtx: &sealevel.SlotCtx{Slot: 260}, + epochRewards: store, + } + address := solana.NewWallet().PublicKey() + missing := solana.NewWallet().PublicKey() + require.NoError(t, server.RecordEpochRewards(&b.Block{ + Slot: 256, Epoch: 2, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 42, PostBalance: 1_042, RewardType: rpc.RewardTypeVoting, + }}, + })) + + confirmed, err := server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String(), missing.String()}, map[string]any{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Len(t, confirmed, 2) + require.Nil(t, confirmed[0], "executed rewards are not confirmed until their state is rooted") + require.Nil(t, confirmed[1]) + + finalized, err := server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "finalized", "epoch": 1}, + })) + require.NoError(t, err) + require.Nil(t, finalized[0]) + _, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "finalized", "epoch": 1, "minContextSlot": 1}, + })) + var contextErr *MinContextSlotNotReachedError + require.ErrorAs(t, err, &contextErr) + require.Zero(t, contextErr.ContextSlot) + + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + finalized, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"epoch": 1}, + })) + require.NoError(t, err) + require.Equal(t, &InflationRewardResp{Epoch: 1, EffectiveSlot: 256, Amount: 42, PostBalance: 1_042}, finalized[0]) + confirmed, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Equal(t, finalized, confirmed) + _, err = server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"commitment": "confirmed", "minContextSlot": 260}, + })) + require.ErrorAs(t, err, &contextErr) + require.Equal(t, uint64(256), contextErr.ContextSlot) +} + +func TestGetInflationRewardRejectsInvalidRequests(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{ + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, + slotCtx: &sealevel.SlotCtx{Slot: 260}, + epochRewards: store, + } + address := solana.NewWallet().PublicKey().String() + + tests := []struct { + name string + params []any + typeOf any + }{ + {name: "null addresses", params: []any{nil}, typeOf: &InvalidParamsError{}}, + {name: "invalid address", params: []any{[]string{"not-a-key"}}, typeOf: &InvalidParamsError{}}, + {name: "processed commitment", params: []any{[]string{address}, map[string]any{"commitment": "processed"}}, typeOf: &InvalidParamsError{}}, + {name: "minimum context", params: []any{[]string{address}, map[string]any{"minContextSlot": 261}}, typeOf: &MinContextSlotNotReachedError{}}, + {name: "too many addresses", params: []any{[]string{address, address, address, address, address, address}}, typeOf: &InvalidParamsError{}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := server.GetInflationReward(context.Background(), rawParams(t, test.params)) + require.Error(t, err) + switch test.typeOf.(type) { + case *InvalidParamsError: + require.IsType(t, &InvalidParamsError{}, err) + case *MinContextSlotNotReachedError: + require.IsType(t, &MinContextSlotNotReachedError{}, err) + } + }) + } +} + +func TestGetInflationRewardAfterSnapshotBeforeReplay(t *testing.T) { + store, err := epochrewards.Open(t.TempDir(), 512) + require.NoError(t, err) + server := &RpcServer{ + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 128}, + epochRewards: store, + } + address := solana.NewWallet().PublicKey() + require.NoError(t, server.RecordEpochRewards(&b.Block{ + Slot: 256, Epoch: 2, + Rewards: []rpc.BlockReward{{ + Pubkey: address, Lamports: 42, PostBalance: 1_042, RewardType: rpc.RewardTypeVoting, + }}, + })) + require.NoError(t, server.PrepareEpochRewards(256)) + require.NoError(t, server.SetRootedEpochRewardsSlot(256)) + + result, err := server.GetInflationReward(context.Background(), rawParams(t, []any{ + []string{address.String()}, map[string]any{"epoch": 1, "minContextSlot": 256}, + })) + require.NoError(t, err) + require.Equal(t, &InflationRewardResp{Epoch: 1, EffectiveSlot: 256, Amount: 42, PostBalance: 1_042}, result[0]) +} + +func TestInflationRewardZeroCommissionJSON(t *testing.T) { + encoded, err := json.Marshal(InflationRewardResp{Epoch: 1, EffectiveSlot: 256, Amount: 0, PostBalance: 1_000}) + require.NoError(t, err) + require.JSONEq(t, `{"epoch":1,"effectiveSlot":256,"amount":0,"postBalance":1000,"commission":null}`, string(encoded)) +} + +func rawParams(t *testing.T, value any) jsonrpc.RawParams { + t.Helper() + encoded, err := json.Marshal(value) + require.NoError(t, err) + return jsonrpc.RawParams(encoded) +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 8a3dd7afe..61958f2e2 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -14,6 +14,8 @@ import ( "time" "github.com/Overclock-Validator/mithril/pkg/accountsdb" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochrewards" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/filecoin-project/go-jsonrpc" @@ -31,6 +33,7 @@ type RpcServer struct { slotCtx *sealevel.SlotCtx slotCtxMu sync.RWMutex genesisHash string + epochRewards *epochrewards.Store leaderTPUCacheMu sync.RWMutex leaderTPUByIdentity map[solana.PublicKey]tpuEndpoint @@ -53,11 +56,57 @@ var supportedRPCMethods = map[string]struct{}{ "getBlockHeight": {}, "getEpochInfo": {}, "getGenesisHash": {}, + "getInflationReward": {}, "getLatestBlockhash": {}, "sendTransaction": {}, "simulateTransaction": {}, } +// EnableEpochRewards attaches durable Alpenglow reward history to the RPC server. +func (rpcServer *RpcServer) EnableEpochRewards(dir string, retentionSlots, rootedSlot uint64) error { + store, err := epochrewards.Open(dir, retentionSlots) + if err != nil { + return err + } + if err := store.SetRooted(rootedSlot); err != nil { + return err + } + rpcServer.epochRewards = store + return nil +} + +// RecordEpochRewards records rewards from a successfully replayed block. +func (rpcServer *RpcServer) RecordEpochRewards(block *b.Block) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + return rpcServer.epochRewards.RecordBlock(block) +} + +// PrepareEpochRewards stages rewards with the matching durable fold. +func (rpcServer *RpcServer) PrepareEpochRewards(through uint64) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + return rpcServer.epochRewards.Prepare(through) +} + +// SetRootedEpochRewardsSlot advances the durable reward watermark. +func (rpcServer *RpcServer) SetRootedEpochRewardsSlot(slot uint64) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + return rpcServer.epochRewards.SetRooted(slot) +} + +// RewindEpochRewards drops rewards from a discarded fork suffix. +func (rpcServer *RpcServer) RewindEpochRewards(fromSlot uint64) error { + if rpcServer == nil || rpcServer.epochRewards == nil { + return nil + } + return rpcServer.epochRewards.Rewind(fromSlot) +} + func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *sealevel.SysvarEpochSchedule, genesisHash solana.Hash) *RpcServer { var err error rpcServer := &RpcServer{genesisHash: genesisHash.String()} diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index a7e2ed934..ee2b33d4e 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -51,6 +51,7 @@ func CleanAccountsDbDir(accountsDbDir string) { // so retaining them would waste space and make stale diagnostics look // actionable. "transaction-status-checkpoints", + "rpc-epoch-rewards", "mithril_db", "mithril_db_log_shards", "bankhash_db", diff --git a/pkg/snapshot/cleanup_test.go b/pkg/snapshot/cleanup_test.go index d7799049f..0df9377ce 100644 --- a/pkg/snapshot/cleanup_test.go +++ b/pkg/snapshot/cleanup_test.go @@ -9,12 +9,16 @@ import ( "github.com/stretchr/testify/require" ) -func TestCleanAccountsDbDirRemovesTransactionStatusCheckpoints(t *testing.T) { +func TestCleanAccountsDbDirRemovesReplaySidecars(t *testing.T) { root := t.TempDir() checkpointDir := filepath.Join(root, "transaction-status-checkpoints") + rewardDir := filepath.Join(root, "rpc-epoch-rewards") require.NoError(t, os.MkdirAll(checkpointDir, 0o755)) + require.NoError(t, os.MkdirAll(rewardDir, 0o755)) require.NoError(t, os.WriteFile(filepath.Join(checkpointDir, "stale.bin"), []byte("stale"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(rewardDir, "stale.json"), []byte("stale"), 0o644)) CleanAccountsDbDir(root) assert.NoDirExists(t, checkpointDir) + assert.NoDirExists(t, rewardDir) }