Add validator metrics RPC methods

README.md

@@ -223,6 +223,12 @@ curl http://YOUR_MITHRIL_IP:8899 -X POST -H "Content-Type: application/json" -d
 - `getBlockHeight` - Get current block height
 - `getEpochInfo` - Get current epoch info
 - `getLatestBlockhash` - Get recent blockhash
+- `getSlot` - Get the slot at the requested commitment (defaults to finalized)
+
+`getSlot` accepts an optional config object with `commitment` and `minContextSlot`.
+On Alpenglow, `processed` uses the live replay slot; `confirmed` and `finalized`
+use the published rooted slot. If that slot is below `minContextSlot`, the request
+returns error `-32016` with the available slot in `data.contextSlot`.
 
 We're actively expanding RPC method coverage. Upcoming methods include transaction simulation, send transaction, and get leader schedule.
 

cmd/mithril/node/checkpoint_recovery_test.go

@@ -134,6 +134,34 @@ func TestRecoveredManifestContextAdvancesStaleStateBeforeIntegrity(t *testing.T)
 	assert.Equal(t, manifestCtx, s.LastRootedContext)
 }
 
+type recordingRPCBankState struct {
+	slot             uint64
+	blockHeight      uint64
+	transactionCount uint64
+}
+
+func (state *recordingRPCBankState) SetRootedBankState(slot, blockHeight, transactionCount uint64) {
+	state.slot = slot
+	state.blockHeight = blockHeight
+	state.transactionCount = transactionCount
+}
+
+func TestPublishRPCBankStateUsesRecoveredRoot(t *testing.T) {
+	transactionCount := uint64(789)
+	server := new(recordingRPCBankState)
+	publishRPCBankState(server, &state.MithrilState{
+		ManifestParentSlot:       10,
+		ManifestBlockHeight:      20,
+		ManifestTransactionCount: 30,
+		LastRootedContext: &state.ResumeContext{
+			Slot:             123,
+			BlockHeight:      456,
+			TransactionCount: &transactionCount,
+		},
+	})
+	assert.Equal(t, recordingRPCBankState{slot: 123, blockHeight: 456, transactionCount: 789}, *server)
+}
+
 func writeCheckpointManifest(t *testing.T, accountsDbPath, manifestsDir string, seq, root, fileID uint64, suffix string) *state.TransactionStatusCheckpointRef {
 	t.Helper()
 	payload := []byte(fmt.Sprintf("checkpoint-payload-%d", root))

cmd/mithril/node/node.go

@@ -2566,6 +2566,7 @@ postBootstrap:
 		klog.Fatalf("invalid port: %d", rpcPort)
 	} else if rpcPort != 0 {
 		rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), epochScheduleFromState(mithrilState), solana.MustHashFromBase58(networkGenesisHash))
+		publishRPCBankState(rpcServer, mithrilState)
 		rpcServer.Start()
 		mlog.Log.Infof("Started RPC server on port %d", rpcPort)
 	}
@@ -4606,6 +4607,7 @@ func runReplayWithRecovery(
 			mlog.Log.Errorf("fork switch: no rooted checkpoint context to re-replay from; halting")
 			break
 		}
+		publishRPCBankState(rpcServer, mithrilState)
 		if mithrilState.LastRootedSlot > prevRooted { // progress since last attempt -> fresh budget
 			attempt = 0
 			prevRooted = mithrilState.LastRootedSlot
@@ -4645,3 +4647,24 @@ func runReplayWithRecovery(
 	}
 	return result
 }
+
+type rpcBankStateSetter interface {
+	SetRootedBankState(slot, blockHeight, transactionCount uint64)
+}
+
+func publishRPCBankState(rpcServer rpcBankStateSetter, mithrilState *state.MithrilState) {
+	if rpcServer == nil || mithrilState == nil {
+		return
+	}
+	rootedSlot := mithrilState.ManifestParentSlot
+	rootedBlockHeight := mithrilState.ManifestBlockHeight
+	rootedTransactionCount := mithrilState.ManifestTransactionCount
+	if rootedCtx := mithrilState.LastRootedContext; rootedCtx != nil {
+		rootedSlot = rootedCtx.Slot
+		rootedBlockHeight = rootedCtx.BlockHeight
+		if rootedCtx.TransactionCount != nil {
+			rootedTransactionCount = *rootedCtx.TransactionCount
+		}
+	}
+	rpcServer.SetRootedBankState(rootedSlot, rootedBlockHeight, rootedTransactionCount)
+}

pkg/accountsdb/accountsdb.go

@@ -34,6 +34,7 @@ type AccountsDb struct {
 	AcctsDir         string
 	LargestFileId    atomic.Uint64
 	VoteAcctCache    otter.Cache[solana.PublicKey, *accounts.Account]
+	voteIndexMu      sync.Mutex
 	CommonAcctsCache otter.Cache[solana.PublicKey, *accounts.Account]
 	ProgramCache     otter.Cache[solana.PublicKey, *ProgramCacheEntry]
 	// Otter permits concurrent ordinary operations but not Clear. Rewind takes
@@ -99,6 +100,11 @@ func (accountsDb *AccountsDb) StoreQueueLen() int {
 	return accountsDb.inProgressStoreRequests.Len()
 }
 
+// DurableThrough returns the highest slot whose fold is fully committed.
+func (accountsDb *AccountsDb) DurableThrough() uint64 {
+	return accountsDb.durableThrough.Load()
+}
+
 // silentLogger implements pebble.Logger but discards all messages.
 // This suppresses verbose WAL recovery messages on startup.
 type silentLogger struct{}

pkg/accountsdb/appendvec_test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/binary"
 	"testing"
 
+	"github.com/Overclock-Validator/mithril/pkg/addresses"
 	"github.com/gagliardetto/solana-go"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -21,7 +22,7 @@ func TestBuildIndexEntriesDoesNotReadPastSliceLen(t *testing.T) {
 	copy(backing[len(real):], phantom)
 	data := backing[:len(real)] // len excludes phantom; cap and manifest include it.
 
-	pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(cap(data)), 7, 9)
+	pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(cap(data)), 7, 9)
 	require.NoError(t, err)
 	require.Equal(t, []solana.PublicKey{realKey}, pubkeys)
 	require.Len(t, entries, 1)
@@ -37,7 +38,7 @@ func TestBuildIndexEntriesBoundsOversizedManifestByDataLen(t *testing.T) {
 	var pubkeys []solana.PublicKey
 	require.NotPanics(t, func() {
 		var err error
-		pubkeys, _, _, err = BuildIndexEntriesFromAppendVecs(data, uint64(len(data)+hdrLen), 8, 10)
+		pubkeys, _, _, _, err = BuildIndexEntriesFromAppendVecs(data, uint64(len(data)+hdrLen), 8, 10)
 		require.NoError(t, err)
 	})
 	require.Equal(t, []solana.PublicKey{key}, pubkeys)
@@ -54,7 +55,7 @@ func TestBuildIndexEntriesStopsAtDefaultZeroLamportTerminator(t *testing.T) {
 	binary.LittleEndian.PutUint64(terminator[dataLenOffset:dataLenOffset+8], ^uint64(0))
 	data := append(append(append([]byte{}, first...), terminator...), afterTerminator...)
 
-	pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 12, 13)
+	pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 12, 13)
 	require.NoError(t, err)
 	require.Equal(t, []solana.PublicKey{firstKey}, pubkeys)
 	require.Len(t, entries, 1)
@@ -67,7 +68,7 @@ func TestBuildIndexEntriesTerminatorRequiresDefaultKeyAndZeroLamports(t *testing
 	zeroLamportAccount := marshalAppendVecTestAccount(t, zeroLamportKey, 0, []byte("tombstone"))
 	data := append(defaultKeyAccount, zeroLamportAccount...)
 
-	pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 18, 19)
+	pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 18, 19)
 	require.NoError(t, err)
 	require.Equal(t, []solana.PublicKey{defaultKey, zeroLamportKey}, pubkeys)
 	require.Len(t, entries, 2)
@@ -80,7 +81,7 @@ func TestBuildIndexEntriesRejectsTruncatedAccountData(t *testing.T) {
 	copy(data[pubkeyOffset:pubkeyOffset+32], key[:])
 	binary.LittleEndian.PutUint64(data[lamportsOffset:lamportsOffset+8], 1)
 
-	pubkeys, entries, stakeEntries, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 14, 15)
+	pubkeys, entries, stakeEntries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 14, 15)
 	require.ErrorContains(t, err, "truncated appendvec account data")
 	assert.Nil(t, pubkeys)
 	assert.Nil(t, entries)
@@ -92,12 +93,23 @@ func TestBuildIndexEntriesAcceptsFinalAccountWithoutAlignmentPadding(t *testing.
 	encoded := marshalAppendVecTestAccount(t, key, 77, []byte{1, 2, 3})
 	data := encoded[: hdrLen+3 : hdrLen+3]
 
-	pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(encoded)), 16, 17)
+	pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(encoded)), 16, 17)
 	require.NoError(t, err)
 	require.Equal(t, []solana.PublicKey{key}, pubkeys)
 	require.Len(t, entries, 1)
 }
 
+func TestBuildIndexEntriesCollectsVoteAccounts(t *testing.T) {
+	vote := appendVecTestPubkey(10)
+	ordinary := appendVecTestPubkey(11)
+	voteData := marshalAppendVecTestAccount(t, vote, 1, nil)
+	copy(voteData[ownerOffset:ownerOffset+32], addresses.VoteProgramAddr[:])
+	data := append(voteData, marshalAppendVecTestAccount(t, ordinary, 1, nil)...)
+	_, _, _, votes, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 20, 21)
+	require.NoError(t, err)
+	require.Equal(t, []solana.PublicKey{vote}, votes)
+}
+
 func appendVecTestPubkey(firstByte byte) solana.PublicKey {
 	var key solana.PublicKey
 	key[0] = firstByte

pkg/accountsdb/compact.go

@@ -268,7 +268,7 @@ func (db *AccountsDb) compactFile(c compactCandidate, minDeadFraction float64) (
 		return false, 0, err
 	}
 
-	pubkeys, idxEntries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), c.slot, c.fileId)
+	pubkeys, idxEntries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), c.slot, c.fileId)
 	if err != nil {
 		return false, 0, err
 	}

pkg/accountsdb/fold.go

@@ -13,6 +13,7 @@ import (
 	"sort"
 
 	"github.com/Overclock-Validator/mithril/pkg/accounts"
+	"github.com/Overclock-Validator/mithril/pkg/addresses"
 	"github.com/cockroachdb/pebble"
 	"golang.org/x/sync/errgroup"
 )
@@ -181,7 +182,10 @@ func (db *AccountsDb) CommitBatch(
 	segErr := func() error {
 		for _, k := range keys {
 			v := union[k]
-			records = append(records, ManifestRecord{Pubkey: k, Offset: dataLen, OwnerSlot: v.ownerSlot})
+			records = append(records, ManifestRecord{
+				Pubkey: k, Offset: dataLen, OwnerSlot: v.ownerSlot,
+				Vote: v.acct.Lamports > 0 && v.acct.Owner == addresses.VoteProgramAddr,
+			})
 			ava := AppendVecAccount{
 				DataLen:    uint64(len(v.acct.Data)),
 				Pubkey:     v.acct.Key,
@@ -304,14 +308,16 @@ func (db *AccountsDb) CommitBatch(
 	// ordinary concurrent-safe cache operations finish.
 	db.refreshReadCacheEntries(live)
 
+	// Advance the watermark before removing pendingFold so rooted readers keep
+	// waiting until the matching bank state is published.
+	db.durableThrough.Store(throughSlot)
 	db.readCacheEpochMu.Lock()
 	db.pendingFold = nil
 	db.readCacheEpochMu.Unlock()
 	fire(db.foldHooks.afterIndexCommit)
 
 	// (8) Publish.
 	db.lastBatchSeq = batchSeq
-	db.durableThrough.Store(throughSlot)
 
 	return BatchCommitResult{
 		BatchSeq:    batchSeq,
@@ -337,6 +343,11 @@ func (db *AccountsDb) applyManifestToIndex(m *SegmentManifest) error {
 		if err := batch.Set(r.Pubkey[:], idxBuf[:], nil); err != nil {
 			return err
 		}
+		if r.Vote {
+			if err := batch.Set(voteIndexKey(r.Pubkey), nil, nil); err != nil {
+				return err
+			}
+		}
 	}
 	if err := batch.Set(metaKeyLastBatch, encodeFoldMeta(foldMeta{
 		BatchSeq:    m.BatchSeq,

pkg/accountsdb/fold_test.go

@@ -116,6 +116,19 @@ func TestCommitBatchReadsBackAndDedupes(t *testing.T) {
 	assert.Equal(t, []byte("ctx-108"), manifest.ResumeCtx)
 }
 
+func TestCommitBatchPublishesDurableThroughWithIndex(t *testing.T) {
+	db, _ := newFoldTestDb(t)
+	defer db.CloseDb()
+
+	db.foldHooks.afterIndexCommit = func() {
+		require.Equal(t, uint64(42), db.DurableThrough())
+	}
+	_, err := db.CommitBatch(foldDeltas(
+		accounts.SlotDelta{Slot: 42, Delta: []*accounts.Account{foldAcct(1, 1, nil)}},
+	), 42, nil, nil)
+	require.NoError(t, err)
+}
+
 // Manifest encode/decode round-trip, and CRC detection of a torn manifest.
 func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) {
 	dir := t.TempDir()
@@ -131,7 +144,7 @@ func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) {
 		Bankhashes:  []SlotBankhash{{Slot: 100, Bankhash: bh(100)}, {Slot: 130, Bankhash: bh(130)}},
 		Records: []ManifestRecord{
 			{Pubkey: [32]byte{1}, Offset: 0, OwnerSlot: 100, PrevValid: true, Prev: AccountIndexEntry{Slot: 90, FileId: 3, Offset: 77}},
-			{Pubkey: [32]byte{2}, Offset: 136, OwnerSlot: 130},
+			{Pubkey: [32]byte{2}, Offset: 136, OwnerSlot: 130, Vote: true},
 		},
 		ResumeCtx: []byte(`{"slot":130}`),
 	}

pkg/accountsdb/index.go

@@ -100,10 +100,12 @@ func WriteStakePubkeyIndex(path string, entries []StakeIndexEntry) error {
 // - pubkeys: all account pubkeys
 // - acctIdxEntries: index entries for each account
 // - stakeEntries: stake account pubkeys with their appendvec location hints
-func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, fileId uint64) ([]solana.PublicKey, []AccountIndexEntry, []StakeIndexEntry, error) {
+// - votePubkeys: vote-program account candidates, including accounts without active stake
+func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, fileId uint64) ([]solana.PublicKey, []AccountIndexEntry, []StakeIndexEntry, []solana.PublicKey, error) {
 	pubkeys := make([]solana.PublicKey, 0, 20000)
 	acctIdxEntries := make([]AccountIndexEntry, 0, 20000)
 	stakeEntries := make([]StakeIndexEntry, 0, 1000)
+	var votePubkeys []solana.PublicKey
 	parser := &appendVecParser{Buf: data, FileSize: fileSize, FileId: fileId, Slot: slot}
 
 	var owner solana.PublicKey
@@ -117,7 +119,7 @@ func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64,
 			if errors.Is(err, io.EOF) {
 				break
 			}
-			return nil, nil, nil, fmt.Errorf("parse appendvec slot=%d file_id=%d: %w", slot, fileId, err)
+			return nil, nil, nil, nil, fmt.Errorf("parse appendvec slot=%d file_id=%d: %w", slot, fileId, err)
 		}
 		// Collect stake account entries with appendvec location hints
 		if bytes.Equal(owner[:], addresses.StakeProgramAddr[:]) {
@@ -128,7 +130,10 @@ func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64,
 				Offset: acctIdxEntries[idx].Offset,
 			})
 		}
+		if owner == addresses.VoteProgramAddr {
+			votePubkeys = append(votePubkeys, pubkeys[len(pubkeys)-1])
+		}
 	}
 
-	return pubkeys, acctIdxEntries, stakeEntries, nil
+	return pubkeys, acctIdxEntries, stakeEntries, votePubkeys, nil
 }

pkg/accountsdb/segment.go

@@ -62,6 +62,7 @@ type ManifestRecord struct {
 	Offset    uint64            // record offset within this segment's data file
 	OwnerSlot uint64            // slot that produced this version (observability)
 	PrevValid bool              // false => the key was absent from the index before this batch
+	Vote      bool              // this version is a nonzero-lamport vote-program account
 	Prev      AccountIndexEntry // index entry before this batch overwrote it
 }
 
@@ -138,11 +139,14 @@ func (m *SegmentManifest) encode() []byte {
 		body.Write(r.Pubkey[:])
 		put64(r.Offset)
 		put64(r.OwnerSlot)
+		var flags byte
 		if r.PrevValid {
-			body.WriteByte(1)
-		} else {
-			body.WriteByte(0)
+			flags |= 1
 		}
+		if r.Vote {
+			flags |= 2
+		}
+		body.WriteByte(flags)
 		r.Prev.Marshal(&prevBuf)
 		body.Write(prevBuf[:])
 	}
@@ -324,7 +328,11 @@ func ReadSegmentManifest(path string) (*SegmentManifest, error) {
 		if err != nil {
 			return nil, err
 		}
-		r.PrevValid = pv == 1
+		if pv&^3 != 0 {
+			return nil, ErrTornManifest
+		}
+		r.PrevValid = pv&1 != 0
+		r.Vote = pv&2 != 0
 		prevRaw, err := d.bytes(24)
 		if err != nil {
 			return nil, err